diff --git a/roles/tools/README.md b/roles/tools/README.md new file mode 100644 index 000000000..dc8bc1754 --- /dev/null +++ b/roles/tools/README.md @@ -0,0 +1,33 @@ +--- +Tools +========= + +This role is designed to configure a server to run the scripts from the surfconext-tool and sram-tools. +The role was created after the server had already been installed, so this role is a recreation of that. +In case you are using this role to install onto a new server make sure everything required is there, test the revelevant scripts etc. + +Requirements +------------ + +None. + +Role Variables +-------------- + +None. + +Dependencies +------------ + +None. + +License +-------------- + +These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE). + +Support +-------------- + +* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) +* Or you can join our [Slack Workspace](https://edu.nl/ocslk) diff --git a/roles/tools/defaults/main.yml b/roles/tools/defaults/main.yml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/roles/tools/defaults/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/tools/files/logos-sync b/roles/tools/files/logos-sync new file mode 100644 index 000000000..775db9614 --- /dev/null +++ b/roles/tools/files/logos-sync @@ -0,0 +1,3 @@ +MAILTO=coin-logs@surf.nl +HOME=/srv/logos +14 * * * * surfconext (cd SURF-member-logos && git pull --rebase --quiet && cd .. && ./sync && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/c0b3b1fd-8d40-4dbd-9027-ec3c42f9da96) | /bin/logger -t surfconext-logo-sync diff --git a/roles/tools/files/sram-managewatch b/roles/tools/files/sram-managewatch new file mode 100644 index 000000000..a8f7a0e62 --- /dev/null +++ b/roles/tools/files/sram-managewatch @@ -0,0 +1,3 @@ +MAILTO=sram-beheer@surf.nl +PATH=/usr/local/bin:/usr/bin:/bin +*/10 8-17 * * 1-5 sram cd /srv/sram/sram-rapportage/ && ./watch-manage.sh -m && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/6ab74c74-8e27-4a58-9774-e6fbae1b7d7c diff --git a/roles/tools/files/sram-rapportage b/roles/tools/files/sram-rapportage new file mode 100644 index 000000000..151fb4567 --- /dev/null +++ b/roles/tools/files/sram-rapportage @@ -0,0 +1,2 @@ +MAILTO=sram-beheer@surf.nl +0 6 1 * * sram cd /srv/sram/sram-rapportage/ && ./sram-report-idps.py $(date --date="yesterday" +\%Y-\%m) diff --git a/roles/tools/files/sync b/roles/tools/files/sync new file mode 100644 index 000000000..83a9ae157 --- /dev/null +++ b/roles/tools/files/sync @@ -0,0 +1,39 @@ +#!/bin/bash + +# based on surfconext-tools/geoip/geoip-update.sh +# +# set -e + +cd /srv/logos + +DIR="SURF-member-logos/200x200/" +SYNC_USER="staticsync" +SYNC_PATH="/opt/openconext/staticdata/data/static/logos/org" +RSYNC_RSH="ssh -o StrictHostKeyChecking=yes" +export RSYNC_RSH + +declare -A sync_hosts +sync_hosts=( + ["test2"]="docker1.test2.ams.surfconext.nl" + ["acc"]="docker1.acc.ams.surfconext.nl docker2.acc.utr.surfconext.nl" + ["prd"]="docker1.prd.ams.surfconext.nl docker2.prd.utr.surfconext.nl docker3.prd.ams.surfconext.nl docker4.prd.utr.surfconext.nl dockerstag1.prd.ams.surfconext.nl" +) + +echo "Synching org logos" +(cd SURF-member-logos/ && git log -1 --format='Last update: %ci (commit %C(auto)%h%Creset by %C(blue)%ce%Creset)') + +for env in "test2" "acc" "prd" +do + echo -n " 🔄️ Syncing to ${env}..." + for host in ${sync_hosts[$env]} + do + rsync -4 --quiet --checksum --delete --delete-excluded --mkpath --archive --exclude '.git/' "$DIR" "${SYNC_USER}@${host}:${SYNC_PATH}" + echo -n "🌟" + done + echo "✅" +done + +echo "🏁 finished" + +exit 0 + diff --git a/roles/tools/files/tools-cert b/roles/tools/files/tools-cert new file mode 100644 index 000000000..c8bc2fdfe --- /dev/null +++ b/roles/tools/files/tools-cert @@ -0,0 +1,2 @@ +MAILTO=cert@surfcert.nl +#59 6 * * * surfconext : CT Log checker hits; cd /srv/tools/surfconext-tools/cert && ./check-ctlogs diff --git a/roles/tools/files/tools-contactexport b/roles/tools/files/tools-contactexport new file mode 100644 index 000000000..72d4a6769 --- /dev/null +++ b/roles/tools/files/tools-contactexport @@ -0,0 +1,2 @@ +MAILTO=coin-logs@surf.nl +36 11 * * * surfconext cd /srv/tools/surfconext-tools/contact_export && ./export_to_csv /srv/tools/www/contacts && ./upload_contacts_to_surfdrive.sh && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/45caa998-c4f9-4e55-8753-6abfbe6b3057 diff --git a/roles/tools/files/tools-crmexport b/roles/tools/files/tools-crmexport new file mode 100644 index 000000000..f1553f2eb --- /dev/null +++ b/roles/tools/files/tools-crmexport @@ -0,0 +1,2 @@ +MAILTO=coin-logs@surf.nl +16 07 * * * surfconext cd /srv/tools/surfconext-tools/stats && ./export-crm.py > /srv/tools/www/crm/crm.csv && ./export-crm.py --members > /srv/tools/www/crm/crm-members.csv && ./export-crm.py --all > /srv/tools/www/crm/crm-all.csv && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/010a6462-dcda-458f-b27d-7c092ea62ba1 diff --git a/roles/tools/files/tools-edugain b/roles/tools/files/tools-edugain new file mode 100644 index 000000000..edb4cb017 --- /dev/null +++ b/roles/tools/files/tools-edugain @@ -0,0 +1,3 @@ +MAILTO=surfconext-beheer@surf.nl +45 04 * * 1-5 surfconext cd /srv/tools/surfconext-tools/edugain && ./check-edugain-entities.py && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/e883a30f-d0c4-4f5c-a4c2-4cf20a616cf2 + diff --git a/roles/tools/files/tools-geoip b/roles/tools/files/tools-geoip new file mode 100644 index 000000000..1f7b19b29 --- /dev/null +++ b/roles/tools/files/tools-geoip @@ -0,0 +1,3 @@ +MAILTO=coin-logs@surf.nl +HOME=/srv/tools/surfconext-tools/geoip +36 */6 * * * surfconext (./update_geoip.sh && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/0cb092ba-5cd8-4c5f-8d07-af17fd67d573) | /bin/logger -t surfconext-geoip diff --git a/roles/tools/files/tools-manage b/roles/tools/files/tools-manage new file mode 100644 index 000000000..17b7cfe78 --- /dev/null +++ b/roles/tools/files/tools-manage @@ -0,0 +1,13 @@ +MAILTO=coin-logs@surf.nl +HOME=/srv/tools/surfconext-tools/manage/ + +23 04 * * * surfconext ./stepup-users > /srv/tools/www/sp/stepup.html && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/1f568a20-59cb-4f41-8b7d-f2913fbdf5b1 + +05 16 * * * surfconext ./edugain-consistency && ./edugain-set-autoconnect && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/d8fc00b3-dcb0-4a75-92b5-71346d1f222a + + +30 07,13 * * * surfconext ./edugain-rns-koppel && ./clarin-koppel && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/15fd4203-3b53-4ab0-8934-5c36fbf9b28b + +MAILTO=surfconext-beheer@surf.nl +33 16 * * * surfconext ./edugain-check && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/5373c35e-f2ca-4d22-bd2e-cd29f669d26e +15 16 * * mon surfconext ./manage-orphan && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/0d673322-2187-4f67-ad7c-f16fc091f3cd diff --git a/roles/tools/files/tools-manage-support b/roles/tools/files/tools-manage-support new file mode 100644 index 000000000..fad59366f --- /dev/null +++ b/roles/tools/files/tools-manage-support @@ -0,0 +1,11 @@ +MAILTO=support@surfconext.nl +HOME=/srv/tools/surfconext-tools/manage/ +00 16 * * 1-5 surfconext ./get-edugain-downstream && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/8306bc58-4ae3-4b77-b5d8-ed70476d5dc7 +14 17 * * 1-5 surfconext ./edugain-rs-missing-sps.sh && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/c0b363c9-1343-43e4-baff-1ed670c3e21b +16 17 * * 1-5 surfconext ./edugain-sps-coco+rs_missing && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/d67dbec0-0940-4233-850d-edd9f9e9994d +11 16 * * 1-5 surfconext ./clarin-missing-sps && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/897da5eb-6e1b-4cf4-a88c-610bd09b619b +36 09 * * 5 surfconext ./unused-idps-4-months && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/75ab797f-7802-4376-aa01-c904b12f697c + +# check for problems in manage +44 04 * * 1-5 surfconext cd /srv/tools/surfconext-tools/manage && ./manage-checker-mail && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/5dd2799a-2bd5-4302-98f8-9d51e92e1330 + diff --git a/roles/tools/files/tools-monitoridp b/roles/tools/files/tools-monitoridp new file mode 100644 index 000000000..142573163 --- /dev/null +++ b/roles/tools/files/tools-monitoridp @@ -0,0 +1,8 @@ +MAILTO=surfconext-beheer@surf.nl +HOME=/srv/tools/surfconext-tools/monitor-idp/ + +# run these during the day to prevent overlap with maintenance windows at IdPs +13 9 * * mon-fri surfconext ./diff-manage-metadata && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/782af100-daaa-4ad2-94cd-7181eb092c80 +13 10 * * mon-fri surfconext ./skew.py && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/782af100-daaa-4ad2-94cd-7181eb092c80 +13 11 * * mon-fri surfconext ./check-idp-metadata-url.py && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/782af100-daaa-4ad2-94cd-7181eb092c80 + diff --git a/roles/tools/files/tools-rapportages b/roles/tools/files/tools-rapportages new file mode 100644 index 000000000..d602c46e9 --- /dev/null +++ b/roles/tools/files/tools-rapportages @@ -0,0 +1,8 @@ +MAILTO=surfconext-beheer@surf.nl + +# rapportage staat uit. Of toch aan? Of toch niet? +#0 3 1 * * surfconext /srv/tools/surfconext-tools/rapportage/vers `date --date="-1 month" +\%Y-\%m` && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/4743f5c9-1219-4adb-922b-8dfa4cb0ab98 + +# doesnt work +#27 2 1 1,4,7,10 * surfconext cd /srv/tools/surfconext-tools/teams-check && ./surfconext-teams-mail + diff --git a/roles/tools/files/tools-sab b/roles/tools/files/tools-sab new file mode 100644 index 000000000..2ef153db1 --- /dev/null +++ b/roles/tools/files/tools-sab @@ -0,0 +1,3 @@ +MAILTO=coin-logs@surf.nl +13 2 * * * surfconext cd /srv/tools/surfconext-tools/sab/ && ./fetch && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/30dc1832-08e6-41cd-a6e6-3df4c7a2ab3a + diff --git a/roles/tools/files/tools-stats b/roles/tools/files/tools-stats new file mode 100644 index 000000000..504a2064f --- /dev/null +++ b/roles/tools/files/tools-stats @@ -0,0 +1,6 @@ +MAILTO=surfconext-beheer@surf.nl +03 23 * * mon surfconext cd /srv/tools/surfconext-tools/stats/ && ./crm_manage_sync.py && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/a980a9ef-f8e6-4284-a67a-be91f7c03dd2 + +MAILTO=coin-logs@surf.nl +15 03 * * sat surfconext cd /srv/tools/surfconext-tools/stats/idp/ && ./manage-fetch-idplist > idps-sso.json && ./idpstats > /srv/tools/www/idp/idpstats.html && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/db4ac7a6-d59f-4aa9-89e6-da1bd86e5ebb +31 16 * * * surfconext cd /srv/tools/surfconext-tools/stats/idp/ && ./manage-fetch-idplist | ./idplist | ssh -q docker2.test.ams.surfconext.nl "cat > /opt/openconext/staticdata/data/support/stats/idplist.html" && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/6e5dea8f-2f7a-43fe-8bf5-2e3723bd88b9 diff --git a/roles/tools/files/tools-vlan-acls b/roles/tools/files/tools-vlan-acls new file mode 100644 index 000000000..c8bc65afa --- /dev/null +++ b/roles/tools/files/tools-vlan-acls @@ -0,0 +1,2 @@ +MAILTO=surfconext-beheer@surf.nl +53 6 * * * surfconext cd /srv/tools/vlan-acls/ && bin/check-acl-updates && curl -fsS --retry 3 -o /dev/null https://ping.ohdear.app/d03c3490-c2cb-4015-af1c-f7782e31f937 diff --git a/roles/tools/tasks/main.yml b/roles/tools/tasks/main.yml new file mode 100644 index 000000000..246fd10d6 --- /dev/null +++ b/roles/tools/tasks/main.yml @@ -0,0 +1,101 @@ +--- +- name: "Add surfconext user" + ansible.builtin.user: + name: "surfconext" + uid: "108" + group: "surfconext" + home: "/srv/tools" + +- name: "Add sram user" + ansible.builtin.user: + name: "sram" + uid: "109" + group: "sram" + home: "/srv/sram" + shell: "/usr/sbin/nologin" + +- name: "Create logos directory" + ansible.builtin.file: + state: "directory" + path: "/srv/logos" + owner: "surfconext" + group: "surfconext" + mode: "0755" + +- name: "Place log sync script" + ansible.builtin.copy: + src: "files/sync" + dest: "/srv/logos/sync" + user: "surfconext" + group: "surfconext" + mode: "0755" + +- name: "Place sram directory" + ansible.builtin.file: + state: "directory" + path: "/srv/sram" + owner: "sram" + group: "sram" + mode: "0755" + +- name: "Install packages" + ansible.builtin.package: + name: + - "php-common" + - "certbot" + - "python3-certbot" + - "python3-apt" + - "python3-bs4" + - "python3-dateutil" + - "python3-mysqldb" + - "python3-pymysql" + - "python3-requests" + - "python3-tzlocal" + - "python3-zeep" + state: "present" + +- name: "Clone logos git repository" + ansible.builtin.git: + repo: "https://github.com/SURFnet/SURF-member-logos.git" + dest: "/srv/logos/SURF-member-logos" + update: "no" + +- name: "Clone tools git repository" + ansible.builtin.git: + repo: "git@gitlab.surf.nl:surfconext/surfconext-tools.git" + dest: "/srv/tools/surfconext-tools" + update: "no" + +- name: "Clone sram git repository" + ansible.builtin.git: + repo: "git@gitlab.surf.nl:sram/sram-tools.git" + dest: "/srv/sram/sram-tools" + update: "no" + +- name: "Make symlink for sram tools" + ansible.builtin.file: + state: "link" + src: "sram-tools/sram-rapportage" + dest: "/srv/sram/sram-rapportage" + +- name: "Place cron jobs" + ansible.builtin.copy: + src: "files/{{ item }}" + dest: "/etc/cron.d/{{ item }}" + mode: "0644" + with_items: + - "logos-sync" + - "sram-managewatch" + - "sram-rapportage" + - "tools-cert" + - "tools-contactexport" + - "tools-crmexport" + - "tools-edugain" + - "tools-geoip" + - "tools-manage" + - "tools-manage-support" + - "tools-monitoridp" + - "tools-rapportages" + - "tools-sab" + - "tools-stats" + - "tools-vlan-acls" diff --git a/roles/tools/vars/main.yml b/roles/tools/vars/main.yml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/roles/tools/vars/main.yml @@ -0,0 +1 @@ +---