ci/stale-cleanup: Add workflow input to limit executions

.github/workflows/cleanup-stale-ec2.yml

@@ -31,7 +31,74 @@ jobs:
           aws-region: ${{ matrix.region }}
 
       - name: Cleanup stale AMI build resources
+        shell: bash
+        timeout-minutes: 60 # AWS creds expire after 1h
         env:
           AWS_MAX_ATTEMPTS: 6
           OK_TO_DESTROY: "true"
-        run: ci/cleanup-stale-ami-build-resources.sh
+        run: ci/cleanup-stale-ami-build-resources.sh 2>&1 | tee "$RUNNER_TEMP/cleanup.log"
+
+      - name: Summarize cleanup
+        if: always()
+        run: ci/summarize-cleanup-log.sh "${{ matrix.region }}" "$RUNNER_TEMP/cleanup.log" > "$RUNNER_TEMP/summary.txt"
+
+      - name: Upload summary
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: cleanup-summary-${{ matrix.region }}
+          path: ${{ runner.temp }}/summary.txt
+          if-no-files-found: ignore
+
+  notify:
+    name: Notify Slack
+    needs: cleanup
+    if: always()
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    permissions:
+      actions: read # to read the cleanup jobs' durations
+      contents: read
+
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          sparse-checkout: ci
+          persist-credentials: false
+
+      - name: Download summaries
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          pattern: cleanup-summary-*
+          path: summaries
+
+      - name: Add job durations
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          shopt -s nullglob
+          # one "region<TAB>seconds" line per cleanup job; a failed lookup just leaves durations as n/a
+          gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs" \
+            --jq '.jobs[] | select(.name | startswith("Delete stale build resources")) | [(.name | capture("\\((?<region>[^)]+)\\)").region), ((.completed_at | fromdate) - (.started_at | fromdate))] | @tsv' \
+            >"$RUNNER_TEMP/durations.tsv" || : >"$RUNNER_TEMP/durations.tsv"
+          summaries=(summaries/*/summary.txt)
+          if ((${#summaries[@]} > 0)); then
+            ci/add-job-durations.sh "$RUNNER_TEMP/durations.tsv" "${summaries[@]}"
+          fi
+
+      - name: Build Slack payload
+        env:
+          CLEANUP_RESULT: ${{ needs.cleanup.result }}
+          SLACK_CHANNEL: C0C16MJ8WHX
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          shopt -s nullglob
+          ci/build-slack-payload.sh summaries/*/summary.txt | tee "$RUNNER_TEMP/payload.json"
+
+      - name: Slack Notification
+        uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
+        with:
+          webhook: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
+          webhook-type: incoming-webhook
+          errors: true
+          payload-file-path: ${{ runner.temp }}/payload.json

ci/add-job-durations.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Appends each region's job duration in seconds as a last tab-separated field to its summary line
+# (see ci/summarize-cleanup-log.sh), editing the summary files in place. Regions without an entry
+# in the durations file get "n/a". Lines that are already "region<TAB>n/a" are left alone.
+#
+# Usage: add-job-durations.sh <durations-file> <summary-file>...
+# The durations file has one "region<TAB>seconds" line per region.
+
+if (($# < 2)); then
+	echo "Usage: $0 <durations-file> <summary-file>..." >&2
+	exit 2
+fi
+
+durations=$1
+shift
+
+for file in "$@"; do
+	tmp=$(mktemp)
+	awk -F'\t' -v OFS='\t' 'NR == FNR { secs[$1] = $2; next } NF > 2 { $0 = $0 OFS (($1 in secs) ? secs[$1] : "n/a") } { print }' "$durations" "$file" >"$tmp"
+	mv "$tmp" "$file"
+done

ci/build-slack-payload.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Builds the Slack message payload (JSON on stdout) for the stale-resource cleanup workflow from the per-region lines written by ci/summarize-cleanup-log.sh
+# When anything was cleaned or failed, the message has a Slack table block with one row per region and one column per resource type.
+#
+# Usage: build-slack-payload.sh [summary-file...]
+# Environment:
+#   RUN_URL (link to the run)
+#   CLEANUP_RESULT (result of the cleanup job)
+#   SLACK_CHANNEL (channel to post to)
+
+: "${CLEANUP_RESULT:?CLEANUP_RESULT must be set}"
+: "${RUN_URL:?RUN_URL must be set}"
+: "${SLACK_CHANNEL:?SLACK_CHANNEL must be set}"
+
+input=""
+if (($# > 0)); then
+	input=$(cat "$@")
+fi
+
+printf '%s\n' "$input" | jq -nR \
+	--arg url "$RUN_URL" \
+	--arg result "$CLEANUP_RESULT" \
+	--arg channel "$SLACK_CHANNEL" '
+def cell($t): {type: "raw_text", text: ($t | tostring)};
+def na: .[1] == "n/a";
+def total: if na then "n/a" else (.[2:9] | map(tonumber) | add) end;
+def duration:
+  if na or (.[11] // "n/a") == "n/a" then "n/a"
+  else (.[11] | tonumber | floor) as $s
+    | ($s / 3600 | floor) as $h
+    | ($s % 3600 / 60 | floor) as $m
+    | ($s % 60 | tostring | if length < 2 then "0" + . else . end) as $sec
+    | if $h > 0 then "\($h)h \($m | tostring | if length < 2 then "0" + . else . end)m \($sec)s" else "\($m)m \($sec)s" end
+  end;
+
+# column headers, kept short so the table fits; the seven values after the region are the
+# resource types in the order ci/summarize-cleanup-log.sh writes them
+def headers: ["region", "testinfra", "Packer", "net ifs", "volumes", "sec groups", "key pairs", "snapshots", "TOTAL", "failed", "ghosts", "to clean", "duration"];
+def region_row:
+  [cell(.[0])]
+  + (if na then [range(headers | length - 1) | cell("n/a")]
+     else [.[2:9][] | cell(.)] + [cell(total), cell(.[1]), cell(.[9]), cell(.[10]), cell(duration)] end);
+
+[inputs | select(length > 0) | split("\t")] | sort_by(.[0]) as $rows
+| ($result != "success") as $jobfailed
+| ($rows | any(.[]; na or ((.[1] | tonumber) > 0))) as $regionfailed
+| ($rows | any(.[]; na or ((.[1] | tonumber) > 0) or (total | tonumber) > 0)) as $active
+| (if $jobfailed or $regionfailed or ($rows | length) == 0 then ":red_circle:" else ":large_green_circle:" end) as $status
+| (if ($rows | length) == 0 then
+     [{type: "section", text: {type: "mrkdwn", text: "No cleanup summaries were produced"}}]
+   elif ($active | not) then
+     [{type: "section", text: {type: "mrkdwn", text: "Nothing to clean up"}}]
+   else
+     [{
+       type: "table",
+       column_settings: ([{align: "left"}] + (headers[1:] | map({align: "right"}))),
+       rows: ([headers | map(cell(.))] + ($rows | map(region_row)))
+     }]
+   end) as $body
+| {
+    text: "Stale build resource cleanup",
+    username: "postgres stale resources janitor",
+    icon_emoji: ":supa-cat:",
+    channel: $channel,
+    blocks: (
+      [{type: "section", text: {type: "mrkdwn", text: "\($status) *Stale build resource cleanup*"}}]
+      + $body
+      + [{type: "context", elements: [{type: "mrkdwn", text: ((if $jobfailed then "Cleanup job result: \($result) · " else "" end) + "<\($url)|View run>")}]}]
+    )
+  }
+'

ci/cleanup-stale-ami-build-resources.sh

@@ -60,20 +60,19 @@ readarray -t ids < <(
 		sort -Vu
 )
 
-max=10
 if ! wait $!; then
 	msg "Encountered at least one error discovering stale executions, proceeding anyway"
 	touch .failed/gather
 fi
 if ((${#ids[@]} == 0)); then
 	msg "No stale Packer or testinfra executions found"
 else
-	msg "${#ids[@]} stale executions found, cleaning up $max"
+	msg "${#ids[@]} stale executions found"
 fi
 
 cleaner=$(dirname "$0")/cleanup-ami-build-resources.sh
 # shellcheck disable=SC2016
-printf '%s\n' "${ids[@]::max}" |
+printf '%s\n' "${ids[@]}" |
 	xargs --no-run-if-empty -P"$(nproc)" -n1 bash -c '
 		set -euo pipefail
 		cleaner=$1

ci/summarize-cleanup-log.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Summarizes the output of cleanup-stale-ami-build-resources.sh for one region as one tab-separated
+# line: region, failed count, the number cleaned per resource type in the fixed order
+#   testinfra instance, Packer instance, network interface, volume, security group, key pair, snapshot
+# then the number of ghost executions and the number of executions still to clean, e.g.
+#   "us-east-1<TAB>1<TAB>0<TAB>2<TAB>0<TAB>0<TAB>0<TAB>10<TAB>0<TAB>4<TAB>0".
+# Resources are counted from the "Deleting ..." / "Terminating ..." log lines, minus any that a
+# "Cleanup failed: ..." line says could not be deleted.
+# A ghost execution is one whose cleanup completed with nothing to delete and no failure. Still to
+# clean is the number of stale executions the job found minus the number it started on.
+# A region whose log is missing or empty is reported as "region<TAB>n/a".
+# ci/add-job-durations.sh appends the job duration and ci/build-slack-payload.sh renders the lines.
+
+region=${1:-}
+log=${2:-}
+if [[ -z $region || -z $log ]]; then
+	echo "Usage: $0 <region> <log-file>" >&2
+	exit 2
+fi
+
+if [[ ! -s $log ]]; then
+	printf '%s\tn/a\n' "$region"
+	exit 0
+fi
+
+awk -v region="$region" '
+function add(kind, n) { count[kind] += n }
+
+# lines from the per-execution cleanup start with the execution id; sub() below rewrites $0
+{ id = $1 }
+
+/^[0-9]+ stale executions found/ { found = $1 }
+/ Cleaning up AMI build resources for execution / { started[id] = 1 }
+/ Packer cleanup complete[[:space:]]*$/ { complete[id] = 1 }
+/Cleanup failed: / { failed++; failed_id[id] = 1 }
+
+/Terminating testinfra instances: / { acted[id] = 1; sub(/.*instances: /, ""); add("testinfra instance", NF); next }
+/Terminating Packer instances: / { acted[id] = 1; sub(/.*instances: /, ""); add("Packer instance", NF); next }
+/Cleanup failed: unable to terminate testinfra instances: / { sub(/.*instances: /, ""); add("testinfra instance", -NF); next }
+/Cleanup failed: unable to terminate Packer instances: / { sub(/.*instances: /, ""); add("Packer instance", -NF); next }
+
+/Deleting network interface: / { acted[id] = 1; add("network interface", 1) }
+/Cleanup failed: unable to delete network interface: / { add("network interface", -1) }
+/Deleting volume: / { acted[id] = 1; add("volume", 1) }
+/Cleanup failed: unable to delete volume: / { add("volume", -1) }
+/Deleting security group: / { acted[id] = 1; add("security group", 1) }
+/Cleanup failed: unable to delete security group: / { add("security group", -1) }
+/Deleting key pair: / { acted[id] = 1; add("key pair", 1) }
+/Cleanup failed: unable to delete key pair: / { add("key pair", -1) }
+/Deleting orphaned snapshot: / { acted[id] = 1; add("snapshot", 1) }
+/Cleanup failed: unable to delete orphaned snapshot: / { add("snapshot", -1) }
+
+END {
+	n = split("testinfra instance,Packer instance,network interface,volume,security group,key pair,snapshot", kinds, ",")
+	line = region "\t" (failed + 0)
+	for (i = 1; i <= n; i++) {
+		c = count[kinds[i]] + 0
+		line = line "\t" (c > 0 ? c : 0)
+	}
+
+	processed = ghosts = 0
+	for (e in started) {
+		processed++
+		if (complete[e] && !acted[e] && !failed_id[e]) ghosts++
+	}
+	still = found > processed ? found - processed : 0
+	print line "\t" ghosts "\t" still
+}
+' "$log"