diff --git a/.github/workflows/cleanup-stale-ec2.yml b/.github/workflows/cleanup-stale-ec2.yml index f548954a16..c24d894da0 100644 --- a/.github/workflows/cleanup-stale-ec2.yml +++ b/.github/workflows/cleanup-stale-ec2.yml @@ -31,7 +31,74 @@ jobs: aws-region: ${{ matrix.region }} - name: Cleanup stale AMI build resources + shell: bash + timeout-minutes: 60 # AWS creds expire after 1h env: AWS_MAX_ATTEMPTS: 6 OK_TO_DESTROY: "true" - run: ci/cleanup-stale-ami-build-resources.sh + run: ci/cleanup-stale-ami-build-resources.sh 2>&1 | tee "$RUNNER_TEMP/cleanup.log" + + - name: Summarize cleanup + if: always() + run: ci/summarize-cleanup-log.sh "${{ matrix.region }}" "$RUNNER_TEMP/cleanup.log" > "$RUNNER_TEMP/summary.txt" + + - name: Upload summary + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cleanup-summary-${{ matrix.region }} + path: ${{ runner.temp }}/summary.txt + if-no-files-found: ignore + + notify: + name: Notify Slack + needs: cleanup + if: always() + runs-on: blacksmith-2vcpu-ubuntu-2404 + permissions: + actions: read # to read the cleanup jobs' durations + contents: read + + steps: + - name: Checkout scripts + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + sparse-checkout: ci + persist-credentials: false + + - name: Download summaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cleanup-summary-* + path: summaries + + - name: Add job durations + env: + GH_TOKEN: ${{ github.token }} + run: | + shopt -s nullglob + # one "regionseconds" line per cleanup job; a failed lookup just leaves durations as n/a + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs" \ + --jq '.jobs[] | select(.name | startswith("Delete stale build resources")) | [(.name | capture("\\((?[^)]+)\\)").region), ((.completed_at | fromdate) - (.started_at | fromdate))] | @tsv' \ + >"$RUNNER_TEMP/durations.tsv" || : >"$RUNNER_TEMP/durations.tsv" + summaries=(summaries/*/summary.txt) + if ((${#summaries[@]} > 0)); then + ci/add-job-durations.sh "$RUNNER_TEMP/durations.tsv" "${summaries[@]}" + fi + + - name: Build Slack payload + env: + CLEANUP_RESULT: ${{ needs.cleanup.result }} + SLACK_CHANNEL: C0C16MJ8WHX + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + shopt -s nullglob + ci/build-slack-payload.sh summaries/*/summary.txt | tee "$RUNNER_TEMP/payload.json" + + - name: Slack Notification + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + webhook: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + webhook-type: incoming-webhook + errors: true + payload-file-path: ${{ runner.temp }}/payload.json diff --git a/ci/add-job-durations.sh b/ci/add-job-durations.sh new file mode 100755 index 0000000000..7190db1cd9 --- /dev/null +++ b/ci/add-job-durations.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Appends each region's job duration in seconds as a last tab-separated field to its summary line +# (see ci/summarize-cleanup-log.sh), editing the summary files in place. Regions without an entry +# in the durations file get "n/a". Lines that are already "regionn/a" are left alone. +# +# Usage: add-job-durations.sh ... +# The durations file has one "regionseconds" line per region. + +if (($# < 2)); then + echo "Usage: $0 ..." >&2 + exit 2 +fi + +durations=$1 +shift + +for file in "$@"; do + tmp=$(mktemp) + awk -F'\t' -v OFS='\t' 'NR == FNR { secs[$1] = $2; next } NF > 2 { $0 = $0 OFS (($1 in secs) ? secs[$1] : "n/a") } { print }' "$durations" "$file" >"$tmp" + mv "$tmp" "$file" +done diff --git a/ci/build-slack-payload.sh b/ci/build-slack-payload.sh new file mode 100755 index 0000000000..91ef61eb2a --- /dev/null +++ b/ci/build-slack-payload.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Builds the Slack message payload (JSON on stdout) for the stale-resource cleanup workflow from the per-region lines written by ci/summarize-cleanup-log.sh +# When anything was cleaned or failed, the message has a Slack table block with one row per region and one column per resource type. +# +# Usage: build-slack-payload.sh [summary-file...] +# Environment: +# RUN_URL (link to the run) +# CLEANUP_RESULT (result of the cleanup job) +# SLACK_CHANNEL (channel to post to) + +: "${CLEANUP_RESULT:?CLEANUP_RESULT must be set}" +: "${RUN_URL:?RUN_URL must be set}" +: "${SLACK_CHANNEL:?SLACK_CHANNEL must be set}" + +input="" +if (($# > 0)); then + input=$(cat "$@") +fi + +printf '%s\n' "$input" | jq -nR \ + --arg url "$RUN_URL" \ + --arg result "$CLEANUP_RESULT" \ + --arg channel "$SLACK_CHANNEL" ' +def cell($t): {type: "raw_text", text: ($t | tostring)}; +def na: .[1] == "n/a"; +def total: if na then "n/a" else (.[2:9] | map(tonumber) | add) end; +def duration: + if na or (.[11] // "n/a") == "n/a" then "n/a" + else (.[11] | tonumber | floor) as $s + | ($s / 3600 | floor) as $h + | ($s % 3600 / 60 | floor) as $m + | ($s % 60 | tostring | if length < 2 then "0" + . else . end) as $sec + | if $h > 0 then "\($h)h \($m | tostring | if length < 2 then "0" + . else . end)m \($sec)s" else "\($m)m \($sec)s" end + end; + +# column headers, kept short so the table fits; the seven values after the region are the +# resource types in the order ci/summarize-cleanup-log.sh writes them +def headers: ["region", "testinfra", "Packer", "net ifs", "volumes", "sec groups", "key pairs", "snapshots", "TOTAL", "failed", "ghosts", "to clean", "duration"]; +def region_row: + [cell(.[0])] + + (if na then [range(headers | length - 1) | cell("n/a")] + else [.[2:9][] | cell(.)] + [cell(total), cell(.[1]), cell(.[9]), cell(.[10]), cell(duration)] end); + +[inputs | select(length > 0) | split("\t")] | sort_by(.[0]) as $rows +| ($result != "success") as $jobfailed +| ($rows | any(.[]; na or ((.[1] | tonumber) > 0))) as $regionfailed +| ($rows | any(.[]; na or ((.[1] | tonumber) > 0) or (total | tonumber) > 0)) as $active +| (if $jobfailed or $regionfailed or ($rows | length) == 0 then ":red_circle:" else ":large_green_circle:" end) as $status +| (if ($rows | length) == 0 then + [{type: "section", text: {type: "mrkdwn", text: "No cleanup summaries were produced"}}] + elif ($active | not) then + [{type: "section", text: {type: "mrkdwn", text: "Nothing to clean up"}}] + else + [{ + type: "table", + column_settings: ([{align: "left"}] + (headers[1:] | map({align: "right"}))), + rows: ([headers | map(cell(.))] + ($rows | map(region_row))) + }] + end) as $body +| { + text: "Stale build resource cleanup", + username: "postgres stale resources janitor", + icon_emoji: ":supa-cat:", + channel: $channel, + blocks: ( + [{type: "section", text: {type: "mrkdwn", text: "\($status) *Stale build resource cleanup*"}}] + + $body + + [{type: "context", elements: [{type: "mrkdwn", text: ((if $jobfailed then "Cleanup job result: \($result) ยท " else "" end) + "<\($url)|View run>")}]}] + ) + } +' diff --git a/ci/cleanup-stale-ami-build-resources.sh b/ci/cleanup-stale-ami-build-resources.sh index 99ee35a6ab..149d3cd959 100755 --- a/ci/cleanup-stale-ami-build-resources.sh +++ b/ci/cleanup-stale-ami-build-resources.sh @@ -60,7 +60,6 @@ readarray -t ids < <( sort -Vu ) -max=10 if ! wait $!; then msg "Encountered at least one error discovering stale executions, proceeding anyway" touch .failed/gather @@ -68,12 +67,12 @@ fi if ((${#ids[@]} == 0)); then msg "No stale Packer or testinfra executions found" else - msg "${#ids[@]} stale executions found, cleaning up $max" + msg "${#ids[@]} stale executions found" fi cleaner=$(dirname "$0")/cleanup-ami-build-resources.sh # shellcheck disable=SC2016 -printf '%s\n' "${ids[@]::max}" | +printf '%s\n' "${ids[@]}" | xargs --no-run-if-empty -P"$(nproc)" -n1 bash -c ' set -euo pipefail cleaner=$1 diff --git a/ci/summarize-cleanup-log.sh b/ci/summarize-cleanup-log.sh new file mode 100755 index 0000000000..129fb8ff63 --- /dev/null +++ b/ci/summarize-cleanup-log.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Summarizes the output of cleanup-stale-ami-build-resources.sh for one region as one tab-separated +# line: region, failed count, the number cleaned per resource type in the fixed order +# testinfra instance, Packer instance, network interface, volume, security group, key pair, snapshot +# then the number of ghost executions and the number of executions still to clean, e.g. +# "us-east-110200010040". +# Resources are counted from the "Deleting ..." / "Terminating ..." log lines, minus any that a +# "Cleanup failed: ..." line says could not be deleted. +# A ghost execution is one whose cleanup completed with nothing to delete and no failure. Still to +# clean is the number of stale executions the job found minus the number it started on. +# A region whose log is missing or empty is reported as "regionn/a". +# ci/add-job-durations.sh appends the job duration and ci/build-slack-payload.sh renders the lines. + +region=${1:-} +log=${2:-} +if [[ -z $region || -z $log ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +if [[ ! -s $log ]]; then + printf '%s\tn/a\n' "$region" + exit 0 +fi + +awk -v region="$region" ' +function add(kind, n) { count[kind] += n } + +# lines from the per-execution cleanup start with the execution id; sub() below rewrites $0 +{ id = $1 } + +/^[0-9]+ stale executions found/ { found = $1 } +/ Cleaning up AMI build resources for execution / { started[id] = 1 } +/ Packer cleanup complete[[:space:]]*$/ { complete[id] = 1 } +/Cleanup failed: / { failed++; failed_id[id] = 1 } + +/Terminating testinfra instances: / { acted[id] = 1; sub(/.*instances: /, ""); add("testinfra instance", NF); next } +/Terminating Packer instances: / { acted[id] = 1; sub(/.*instances: /, ""); add("Packer instance", NF); next } +/Cleanup failed: unable to terminate testinfra instances: / { sub(/.*instances: /, ""); add("testinfra instance", -NF); next } +/Cleanup failed: unable to terminate Packer instances: / { sub(/.*instances: /, ""); add("Packer instance", -NF); next } + +/Deleting network interface: / { acted[id] = 1; add("network interface", 1) } +/Cleanup failed: unable to delete network interface: / { add("network interface", -1) } +/Deleting volume: / { acted[id] = 1; add("volume", 1) } +/Cleanup failed: unable to delete volume: / { add("volume", -1) } +/Deleting security group: / { acted[id] = 1; add("security group", 1) } +/Cleanup failed: unable to delete security group: / { add("security group", -1) } +/Deleting key pair: / { acted[id] = 1; add("key pair", 1) } +/Cleanup failed: unable to delete key pair: / { add("key pair", -1) } +/Deleting orphaned snapshot: / { acted[id] = 1; add("snapshot", 1) } +/Cleanup failed: unable to delete orphaned snapshot: / { add("snapshot", -1) } + +END { + n = split("testinfra instance,Packer instance,network interface,volume,security group,key pair,snapshot", kinds, ",") + line = region "\t" (failed + 0) + for (i = 1; i <= n; i++) { + c = count[kinds[i]] + 0 + line = line "\t" (c > 0 ? c : 0) + } + + processed = ghosts = 0 + for (e in started) { + processed++ + if (complete[e] && !acted[e] && !failed_id[e]) ghosts++ + } + still = found > processed ? found - processed : 0 + print line "\t" ghosts "\t" still +} +' "$log"