From fb503fa21069f174ed0c07947e923cc0c270ad00 Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 12:34:39 +0200 Subject: [PATCH 1/3] ci: notify Slack when release jobs fail, are cancelled, or never run --- .../release-failure-notification.yml | 56 +++++++++++ .github/workflows/release.yml | 21 ++++ RELEASING.md | 10 ++ scripts/release-failure-notification.cjs | 98 +++++++++++++++++++ 4 files changed, 185 insertions(+) create mode 100644 .github/workflows/release-failure-notification.yml create mode 100644 scripts/release-failure-notification.cjs diff --git a/.github/workflows/release-failure-notification.yml b/.github/workflows/release-failure-notification.yml new file mode 100644 index 000000000..856efac03 --- /dev/null +++ b/.github/workflows/release-failure-notification.yml @@ -0,0 +1,56 @@ +name: Release failure notification + +on: + workflow_run: + workflows: [Release] + types: [completed] + branches: [main] + +permissions: + actions: read + contents: read + +jobs: + notify-failed: + name: Notify Slack - Failed + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # This workflow must survive cancellation or runner startup failure in Release. + # Never execute code from the triggering run's head in this privileged workflow. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Download approval thread timestamps + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: release-slack-thread-* + path: ${{ runner.temp }}/release-slack-threads + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Check release jobs + id: release-status + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SLACK_CHANNEL_ID: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} + THREAD_DIRECTORY: ${{ runner.temp }}/release-slack-threads + with: + retries: 3 + script: | + const notification = require('./scripts/release-failure-notification.cjs'); + const payload = await notification({ github, context, channel: process.env.SLACK_CHANNEL_ID, threadDirectory: process.env.THREAD_DIRECTORY }); + if (payload) core.setOutput('payload', JSON.stringify(payload)); + + - name: Notify Slack - Failed + if: steps.release-status.outputs.payload != '' + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} + errors: true + retries: 5 + payload: ${{ steps.release-status.outputs.payload }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f644b94f9..8134f6020 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,27 @@ jobs: slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} posthog_project_api_key: ${{ secrets.POSTHOG_PROJECT_API_KEY }} + preserve-slack-thread: + name: Preserve Slack thread + needs: notify-approval-needed + if: needs.notify-approval-needed.outputs.slack_ts != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: {} + steps: + - name: Save approval thread timestamp + env: + THREAD_TS: ${{ needs.notify-approval-needed.outputs.slack_ts }} + run: printf '%s' "$THREAD_TS" > thread-ts.txt + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-slack-thread-${{ github.run_attempt }} + path: thread-ts.txt + retention-days: 30 + if-no-files-found: error + # Bump versions, commit to main, and regenerate references. This is the single # approval-gated job (environment "Release"). It does NOT publish — publishing # happens in the `publish` matrix job below, which has no environment so the diff --git a/RELEASING.md b/RELEASING.md index 568e9dd82..e860094a5 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -27,3 +27,13 @@ You can also trigger a release manually via the workflow's `workflow_dispatch` t > [!IMPORTANT] > Changesets must live in **`.sampo/changesets/`** (this is where `sampo add` puts them). Do **not** create them in the legacy `.changeset/` directory — Sampo and the `Release` workflow only read `.sampo/changesets/*.md`, so a changeset placed anywhere else is silently ignored and **no release is triggered**. + +## Failed or cancelled release + +The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and a publish job (`Publish posthog`, `Publish posthoganalytics`, or `Publish openfeature-provider-posthog`) was never created, for example when GitHub Actions stops a run before the publish jobs start. Once every publish job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. + +Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed. + +Job inspection retries transient GitHub API failures up to three times. If inspection still fails, it logs the HTTP status or error type and message, then posts a fallback alert with the overall run status and logs link so the release can be checked manually. Error diagnostics stay in the run log, not in the Slack message. + +Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run; local tests do not send Slack messages. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs new file mode 100644 index 000000000..26a327afe --- /dev/null +++ b/scripts/release-failure-notification.cjs @@ -0,0 +1,98 @@ +const { readdirSync, readFileSync } = require('node:fs'); +const { join } = require('node:path'); + +const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']); +const publishJobs = ['Publish posthog', 'Publish posthoganalytics', 'Publish openfeature-provider-posthog']; + +const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`); + +function readThreadTimestamp(directory, attempt) { + if (!directory) return undefined; + try { + // Failed-job reruns reuse the approval thread from the latest earlier attempt. + const attempts = readdirSync(directory) + .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1]) + .filter(value => value && Number(value) <= attempt) + .sort((a, b) => Number(b) - Number(a)); + if (attempts.length === 0) return undefined; + const timestamp = readFileSync(join(directory, `release-slack-thread-${attempts[0]}`, 'thread-ts.txt'), 'utf8').trim(); + return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined; + } catch (error) { + if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message); + return undefined; + } +} + +module.exports = async function releaseFailureNotification({ github, context, channel, threadDirectory }) { + const run = context.payload.workflow_run; + const threadTs = readThreadTimestamp(threadDirectory, run.run_attempt); + const destination = { channel, ...(threadTs ? { thread_ts: threadTs } : {}) }; + let jobs; + try { + jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { + ...context.repo, + run_id: run.id, + attempt_number: run.run_attempt, + per_page: 100, + }); + } catch (error) { + console.error('Failed to inspect release jobs:', error.status ?? error.name, error.message); + return { + ...destination, + text: [ + `⚠️ Could not inspect release jobs for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`, + 'Please check this release manually; individual job results are unavailable.', + `<${run.html_url}|View release logs> (run: ${run.conclusion})`, + ].join('\n'), + unfurl_links: false, + unfurl_media: false, + }; + } + + if (jobs.length === 0 && run.conclusion === 'cancelled') { + return null; + } + + const failedJobs = jobs.filter(job => unsuccessful.has(job.conclusion)); + const versionBump = jobs.find(job => job.name === 'Bump versions and commit to main'); + if (run.conclusion === 'failure' && failedJobs.length === 1 + && versionBump?.conclusion === 'failure' && versionBump.steps?.length === 0 + && jobs.some(job => job.name === 'Notify Slack - Rejected' + && job.steps?.some(step => step.name === 'Notify Slack - Rejected' && step.conclusion === 'success'))) { + return null; + } + + const pypiPublished = publishJobs.every(name => { + const cells = jobs.filter(job => isJob(job, name)); + return cells.length > 0 && cells.every(job => job.conclusion === 'success'); + }); + const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released'); + const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`); + + if (run.conclusion !== 'success' && failedJobs.length === 0) { + for (const name of publishJobs) { + if (!jobs.some(job => isJob(job, name))) problems.push(`• ${name}: never ran`); + } + } + + if (pypiPublished && (!releasedNotification || releasedNotification.conclusion === 'skipped')) { + problems.push('• The released Slack notification did not run.'); + } + + if (problems.length === 0 && run.conclusion === 'success') { + return null; + } + + return { + ...destination, + text: [ + pypiPublished + ? `⚠️ PyPI packages published; follow-up problems for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).` + : `❌ Release process failed or was cancelled for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`, + ...problems, + `<${run.html_url}|View release logs> (run: ${run.conclusion})`, + ].join('\n'), + unfurl_links: false, + unfurl_media: false, + }; +}; From 11431956b4cc5e8ffcb5c4e571dd81f82f0a8c8a Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 15:20:42 +0200 Subject: [PATCH 2/3] ci: drop in-job Slack failure replies now covered by the notifier --- .github/workflows/release.yml | 20 -------------------- RELEASING.md | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8134f6020..e7d0fc916 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -250,16 +250,6 @@ jobs: "alertContext": "The version bump/changelog step failed before packages were published." } - - name: Notify Slack - Failed - if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }} - uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main - with: - slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} - slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} - thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }} - message: "❌ Failed to bump versions for `posthog-python@${{ steps.sampo-release.outputs.new_version }}`! " - emoji_reaction: "x" - # Build, publish, and tag each package whose version was bumped in this release. # A matrix over the publishable packages (N packages, like the posthog-ruby and # JS monorepos). The approval gate lives on `version-bump`, so this job has no @@ -457,16 +447,6 @@ jobs: "alertContext": "The package publish step failed." } - - name: Notify Slack - Failed - if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }} - uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main - with: - slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} - slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} - thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }} - message: "❌ Failed to publish `${{ matrix.package.name }}@${{ needs.version-bump.outputs.new_version }}`! " - emoji_reaction: "x" - # Side-effect-free rehearsal: runs ONLY when this workflow is manually # dispatched on a non-main branch (e.g. this PR branch). It applies the Sampo # version bump, builds openfeature-provider-posthog, and publishes it to diff --git a/RELEASING.md b/RELEASING.md index e860094a5..e87bbccdc 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -36,4 +36,4 @@ Cancelled runs with no jobs are ignored, including superseded queued runs and ma Job inspection retries transient GitHub API failures up to three times. If inspection still fails, it logs the HTTP status or error type and message, then posts a fallback alert with the overall run status and logs link so the release can be checked manually. Error diagnostics stay in the run log, not in the Slack message. -Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run; local tests do not send Slack messages. +Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run. From a1f4d796b5cececaed0e85c08f86dfd6c6f68a86 Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 16:27:48 +0200 Subject: [PATCH 3/3] ci: accept a lone thread artifact and gate the notifier on trusted runs --- .../workflows/release-failure-notification.yml | 3 +++ RELEASING.md | 2 +- scripts/release-failure-notification.cjs | 17 +++++++++++------ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-failure-notification.yml b/.github/workflows/release-failure-notification.yml index 856efac03..caec2e759 100644 --- a/.github/workflows/release-failure-notification.yml +++ b/.github/workflows/release-failure-notification.yml @@ -14,6 +14,9 @@ jobs: notify-failed: name: Notify Slack - Failed runs-on: ubuntu-latest + if: >- + github.event.workflow_run.head_repository.full_name == github.repository && + (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') timeout-minutes: 10 steps: # This workflow must survive cancellation or runner startup failure in Release. diff --git a/RELEASING.md b/RELEASING.md index e87bbccdc..b2f7d9cd7 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -30,7 +30,7 @@ You can also trigger a release manually via the workflow's `workflow_dispatch` t ## Failed or cancelled release -The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and a publish job (`Publish posthog`, `Publish posthoganalytics`, or `Publish openfeature-provider-posthog`) was never created, for example when GitHub Actions stops a run before the publish jobs start. Once every publish job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. +The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and a publish job (`Publish posthog`, `Publish posthoganalytics`, or `Publish openfeature-provider-posthog`) was never created, for example when GitHub Actions stops a run before the publish jobs start. Once every publish job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository. Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs index 26a327afe..8f5c9b265 100644 --- a/scripts/release-failure-notification.cjs +++ b/scripts/release-failure-notification.cjs @@ -9,13 +9,18 @@ const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} ( function readThreadTimestamp(directory, attempt) { if (!directory) return undefined; try { + const entries = readdirSync(directory); + // A single matching artifact is extracted straight into the directory. // Failed-job reruns reuse the approval thread from the latest earlier attempt. - const attempts = readdirSync(directory) - .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1]) - .filter(value => value && Number(value) <= attempt) - .sort((a, b) => Number(b) - Number(a)); - if (attempts.length === 0) return undefined; - const timestamp = readFileSync(join(directory, `release-slack-thread-${attempts[0]}`, 'thread-ts.txt'), 'utf8').trim(); + const files = entries.includes('thread-ts.txt') + ? [join(directory, 'thread-ts.txt')] + : entries + .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1]) + .filter(value => value && Number(value) <= attempt) + .sort((a, b) => Number(b) - Number(a)) + .map(value => join(directory, `release-slack-thread-${value}`, 'thread-ts.txt')); + if (files.length === 0) return undefined; + const timestamp = readFileSync(files[0], 'utf8').trim(); return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined; } catch (error) { if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message);