Feature/tools geautomatiseerd

roles/tools/README.md

@@ -0,0 +1,33 @@
+---
+Tools
+=========
+
+This role is designed to configure a server to run the scripts from the surfconext-tool and sram-tools.
+The role was created after the server had already been installed, so this role is a recreation of that.
+In case you are using this role to install onto a new server make sure everything required is there, test the revelevant scripts etc.
+
+Requirements
+------------
+
+None.
+
+Role Variables
+--------------
+
+None.
+
+Dependencies
+------------
+
+None.
+
+License
+--------------
+
+These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE).
+
+Support
+--------------
+
+* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/)
+* Or you can join our [Slack Workspace](https://edu.nl/ocslk)

roles/tools/defaults/main.yml

@@ -0,0 +1 @@
+---

roles/tools/files/logos-sync

@@ -0,0 +1,3 @@
+MAILTO=coin-logs@surf.nl
+HOME=/srv/logos
+14 *     * *   *   surfconext   (cd SURF-member-logos && git pull --rebase --quiet && cd .. && ./sync && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/c0b3b1fd-8d40-4dbd-9027-ec3c42f9da96) | /bin/logger -t surfconext-logo-sync

roles/tools/files/sram-managewatch

@@ -0,0 +1,3 @@
+MAILTO=sram-beheer@surf.nl
+PATH=/usr/local/bin:/usr/bin:/bin
+*/10 8-17	* * 1-5	sram	cd /srv/sram/sram-rapportage/ && ./watch-manage.sh -m && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/6ab74c74-8e27-4a58-9774-e6fbae1b7d7c

roles/tools/files/sram-rapportage

@@ -0,0 +1,2 @@
+MAILTO=sram-beheer@surf.nl
+0 6	1 * *	sram	cd /srv/sram/sram-rapportage/ && ./sram-report-idps.py $(date --date="yesterday" +\%Y-\%m)

roles/tools/files/sync

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# based on surfconext-tools/geoip/geoip-update.sh
+#
+# set -e
+
+cd /srv/logos
+
+DIR="SURF-member-logos/200x200/"
+SYNC_USER="staticsync"
+SYNC_PATH="/opt/openconext/staticdata/data/static/logos/org"
+RSYNC_RSH="ssh -o StrictHostKeyChecking=yes"
+export RSYNC_RSH
+
+declare -A sync_hosts
+sync_hosts=(
+    ["test2"]="docker1.test2.ams.surfconext.nl"
+    ["acc"]="docker1.acc.ams.surfconext.nl docker2.acc.utr.surfconext.nl"
+    ["prd"]="docker1.prd.ams.surfconext.nl docker2.prd.utr.surfconext.nl docker3.prd.ams.surfconext.nl docker4.prd.utr.surfconext.nl dockerstag1.prd.ams.surfconext.nl"
+)
+
+echo "Synching org logos"
+(cd SURF-member-logos/ && git log -1 --format='Last update: %ci (commit %C(auto)%h%Creset by %C(blue)%ce%Creset)')
+
+for env in "test2" "acc" "prd"
+do
+    echo -n "  🔄️ Syncing to ${env}..."
+    for host in ${sync_hosts[$env]}
+    do
+        rsync -4 --quiet --checksum --delete --delete-excluded --mkpath --archive --exclude '.git/' "$DIR" "${SYNC_USER}@${host}:${SYNC_PATH}"
+        echo -n "🌟"
+    done
+    echo "✅"
+done
+
+echo "🏁 finished"
+
+exit 0
+

roles/tools/files/tools-cert

@@ -0,0 +1,2 @@
+MAILTO=cert@surfcert.nl
+#59 6 * * * surfconext : CT Log checker hits; cd /srv/tools/surfconext-tools/cert && ./check-ctlogs

roles/tools/files/tools-contactexport

@@ -0,0 +1,2 @@
+MAILTO=coin-logs@surf.nl
+36 11	* * *	surfconext	cd /srv/tools/surfconext-tools/contact_export && ./export_to_csv /srv/tools/www/contacts && ./upload_contacts_to_surfdrive.sh && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/45caa998-c4f9-4e55-8753-6abfbe6b3057

roles/tools/files/tools-crmexport

@@ -0,0 +1,2 @@
+MAILTO=coin-logs@surf.nl
+16 07	* * *	surfconext	cd /srv/tools/surfconext-tools/stats && ./export-crm.py > /srv/tools/www/crm/crm.csv && ./export-crm.py --members > /srv/tools/www/crm/crm-members.csv && ./export-crm.py --all > /srv/tools/www/crm/crm-all.csv && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/010a6462-dcda-458f-b27d-7c092ea62ba1

roles/tools/files/tools-edugain

@@ -0,0 +1,3 @@
+MAILTO=surfconext-beheer@surf.nl
+45 04	* * 1-5   	surfconext  	cd /srv/tools/surfconext-tools/edugain && ./check-edugain-entities.py && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/e883a30f-d0c4-4f5c-a4c2-4cf20a616cf2
+

roles/tools/files/tools-geoip

@@ -0,0 +1,3 @@
+MAILTO=coin-logs@surf.nl
+HOME=/srv/tools/surfconext-tools/geoip
+36 */6     * *   *   surfconext   (./update_geoip.sh && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/0cb092ba-5cd8-4c5f-8d07-af17fd67d573) | /bin/logger -t surfconext-geoip

roles/tools/files/tools-manage

@@ -0,0 +1,13 @@
+MAILTO=coin-logs@surf.nl
+HOME=/srv/tools/surfconext-tools/manage/
+
+23 04     * *   *   surfconext   ./stepup-users > /srv/tools/www/sp/stepup.html && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/1f568a20-59cb-4f41-8b7d-f2913fbdf5b1
+
+05 16     * *   *   surfconext   ./edugain-consistency && ./edugain-set-autoconnect && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/d8fc00b3-dcb0-4a75-92b5-71346d1f222a
+
+
+30 07,13  * *   *   surfconext   ./edugain-rns-koppel && ./clarin-koppel && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/15fd4203-3b53-4ab0-8934-5c36fbf9b28b
+
+MAILTO=surfconext-beheer@surf.nl
+33 16     * *   *   surfconext   ./edugain-check  && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/5373c35e-f2ca-4d22-bd2e-cd29f669d26e
+15 16     * *   mon surfconext   ./manage-orphan && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/0d673322-2187-4f67-ad7c-f16fc091f3cd

roles/tools/files/tools-manage-support

@@ -0,0 +1,11 @@
+MAILTO=support@surfconext.nl
+HOME=/srv/tools/surfconext-tools/manage/
+00 16  * *  1-5   surfconext   ./get-edugain-downstream            && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/8306bc58-4ae3-4b77-b5d8-ed70476d5dc7
+14 17  * *  1-5   surfconext   ./edugain-rs-missing-sps.sh         && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/c0b363c9-1343-43e4-baff-1ed670c3e21b
+16 17  * *  1-5   surfconext   ./edugain-sps-coco+rs_missing       && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/d67dbec0-0940-4233-850d-edd9f9e9994d
+11 16  * *  1-5   surfconext   ./clarin-missing-sps                && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/897da5eb-6e1b-4cf4-a88c-610bd09b619b
+36 09  * *  5     surfconext   ./unused-idps-4-months              && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/75ab797f-7802-4376-aa01-c904b12f697c
+
+# check for problems in manage
+44 04  * *  1-5   surfconext   cd /srv/tools/surfconext-tools/manage && ./manage-checker-mail && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/5dd2799a-2bd5-4302-98f8-9d51e92e1330
+

roles/tools/files/tools-monitoridp

@@ -0,0 +1,8 @@
+MAILTO=surfconext-beheer@surf.nl
+HOME=/srv/tools/surfconext-tools/monitor-idp/
+
+# run these during the day to prevent overlap with maintenance windows at IdPs
+13  9  * *  mon-fri  surfconext  ./diff-manage-metadata      && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/782af100-daaa-4ad2-94cd-7181eb092c80
+13 10  * *  mon-fri  surfconext  ./skew.py                   && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/782af100-daaa-4ad2-94cd-7181eb092c80
+13 11  * *  mon-fri  surfconext  ./check-idp-metadata-url.py && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/782af100-daaa-4ad2-94cd-7181eb092c80
+

roles/tools/files/tools-rapportages

@@ -0,0 +1,8 @@
+MAILTO=surfconext-beheer@surf.nl
+
+# rapportage staat uit.  Of toch aan?  Of toch niet?
+#0 3 1 * * surfconext /srv/tools/surfconext-tools/rapportage/vers `date --date="-1 month" +\%Y-\%m` && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/4743f5c9-1219-4adb-922b-8dfa4cb0ab98
+
+# doesnt work
+#27 2 1 1,4,7,10 *   surfconext cd /srv/tools/surfconext-tools/teams-check && ./surfconext-teams-mail
+

roles/tools/files/tools-sab

@@ -0,0 +1,3 @@
+MAILTO=coin-logs@surf.nl
+13 2 * * * surfconext cd /srv/tools/surfconext-tools/sab/ && ./fetch && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/30dc1832-08e6-41cd-a6e6-3df4c7a2ab3a
+

roles/tools/files/tools-stats

@@ -0,0 +1,6 @@
+MAILTO=surfconext-beheer@surf.nl
+03 23	* * mon   	surfconext  	cd /srv/tools/surfconext-tools/stats/     && ./crm_manage_sync.py && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/a980a9ef-f8e6-4284-a67a-be91f7c03dd2
+
+MAILTO=coin-logs@surf.nl
+15 03	* * sat 	surfconext  	cd /srv/tools/surfconext-tools/stats/idp/ && ./manage-fetch-idplist > idps-sso.json && ./idpstats > /srv/tools/www/idp/idpstats.html && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/db4ac7a6-d59f-4aa9-89e6-da1bd86e5ebb
+31 16	* * *   	surfconext  	cd /srv/tools/surfconext-tools/stats/idp/ && ./manage-fetch-idplist | ./idplist | ssh -q docker2.test.ams.surfconext.nl "cat > /opt/openconext/staticdata/data/support/stats/idplist.html" && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/6e5dea8f-2f7a-43fe-8bf5-2e3723bd88b9

roles/tools/files/tools-vlan-acls

@@ -0,0 +1,2 @@
+MAILTO=surfconext-beheer@surf.nl
+53 6  * * * surfconext cd /srv/tools/vlan-acls/ && bin/check-acl-updates && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/d03c3490-c2cb-4015-af1c-f7782e31f937

roles/tools/tasks/main.yml

@@ -0,0 +1,101 @@
+---
+- name: "Add surfconext user"
+  ansible.builtin.user:
+    name: "surfconext"
+    uid: "108"
+    group: "surfconext"
+    home: "/srv/tools"
+
+- name: "Add sram user"
+  ansible.builtin.user:
+    name: "sram"
+    uid: "109"
+    group: "sram"
+    home: "/srv/sram"
+    shell: "/usr/sbin/nologin"
+
+- name: "Create logos directory"
+  ansible.builtin.file:
+    state: "directory"
+    path: "/srv/logos"
+    owner: "surfconext"
+    group: "surfconext"
+    mode: "0755"
+
+- name: "Place log sync script"
+  ansible.builtin.copy:
+    src: "files/sync"
+    dest: "/srv/logos/sync"
+    user: "surfconext"
+    group: "surfconext"
+    mode: "0755"
+
+- name: "Place sram directory"
+  ansible.builtin.file:
+    state: "directory"
+    path: "/srv/sram"
+    owner: "sram"
+    group: "sram"
+    mode: "0755"
+
+- name: "Install packages"
+  ansible.builtin.package:
+    name:
+      - "php-common"
+      - "certbot"
+      - "python3-certbot"
+      - "python3-apt"
+      - "python3-bs4"
+      - "python3-dateutil"
+      - "python3-mysqldb"
+      - "python3-pymysql"
+      - "python3-requests"
+      - "python3-tzlocal"
+      - "python3-zeep"
+    state: "present"
+
+- name: "Clone logos git repository"
+  ansible.builtin.git:
+    repo: "https://github.com/SURFnet/SURF-member-logos.git"
+    dest: "/srv/logos/SURF-member-logos"
+    update: "no"
+
+- name: "Clone tools git repository"
+  ansible.builtin.git:
+    repo: "git@gitlab.surf.nl:surfconext/surfconext-tools.git"
+    dest: "/srv/tools/surfconext-tools"
+    update: "no"
+
+- name: "Clone sram git repository"
+  ansible.builtin.git:
+    repo: "git@gitlab.surf.nl:sram/sram-tools.git"
+    dest: "/srv/sram/sram-tools"
+    update: "no"
+
+- name: "Make symlink for sram tools"
+  ansible.builtin.file:
+    state: "link"
+    src: "sram-tools/sram-rapportage"
+    dest: "/srv/sram/sram-rapportage"
+
+- name: "Place cron jobs"
+  ansible.builtin.copy:
+    src: "files/{{ item }}"
+    dest: "/etc/cron.d/{{ item }}"
+    mode: "0644"
+  with_items:
+    - "logos-sync"
+    - "sram-managewatch"
+    - "sram-rapportage"
+    - "tools-cert"
+    - "tools-contactexport"
+    - "tools-crmexport"
+    - "tools-edugain"
+    - "tools-geoip"
+    - "tools-manage"
+    - "tools-manage-support"
+    - "tools-monitoridp"
+    - "tools-rapportages"
+    - "tools-sab"
+    - "tools-stats"
+    - "tools-vlan-acls"

roles/tools/vars/main.yml

@@ -0,0 +1 @@
+---