diff --git a/.github/workflows/release-failure-notification.yml b/.github/workflows/release-failure-notification.yml new file mode 100644 index 000000000..caec2e759 --- /dev/null +++ b/.github/workflows/release-failure-notification.yml @@ -0,0 +1,59 @@ +name: Release failure notification + +on: + workflow_run: + workflows: [Release] + types: [completed] + branches: [main] + +permissions: + actions: read + contents: read + +jobs: + notify-failed: + name: Notify Slack - Failed + runs-on: ubuntu-latest + if: >- + github.event.workflow_run.head_repository.full_name == github.repository && + (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') + timeout-minutes: 10 + steps: + # This workflow must survive cancellation or runner startup failure in Release. + # Never execute code from the triggering run's head in this privileged workflow. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Download approval thread timestamps + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: release-slack-thread-* + path: ${{ runner.temp }}/release-slack-threads + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Check release jobs + id: release-status + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SLACK_CHANNEL_ID: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} + THREAD_DIRECTORY: ${{ runner.temp }}/release-slack-threads + with: + retries: 3 + script: | + const notification = require('./scripts/release-failure-notification.cjs'); + const payload = await notification({ github, context, channel: process.env.SLACK_CHANNEL_ID, threadDirectory: process.env.THREAD_DIRECTORY }); + if (payload) core.setOutput('payload', JSON.stringify(payload)); + + - name: Notify Slack - Failed + if: steps.release-status.outputs.payload != '' + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} + errors: true + retries: 5 + payload: ${{ steps.release-status.outputs.payload }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f644b94f9..e7d0fc916 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,27 @@ jobs: slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} posthog_project_api_key: ${{ secrets.POSTHOG_PROJECT_API_KEY }} + preserve-slack-thread: + name: Preserve Slack thread + needs: notify-approval-needed + if: needs.notify-approval-needed.outputs.slack_ts != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: {} + steps: + - name: Save approval thread timestamp + env: + THREAD_TS: ${{ needs.notify-approval-needed.outputs.slack_ts }} + run: printf '%s' "$THREAD_TS" > thread-ts.txt + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-slack-thread-${{ github.run_attempt }} + path: thread-ts.txt + retention-days: 30 + if-no-files-found: error + # Bump versions, commit to main, and regenerate references. This is the single # approval-gated job (environment "Release"). It does NOT publish — publishing # happens in the `publish` matrix job below, which has no environment so the @@ -229,16 +250,6 @@ jobs: "alertContext": "The version bump/changelog step failed before packages were published." } - - name: Notify Slack - Failed - if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }} - uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main - with: - slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} - slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} - thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }} - message: "❌ Failed to bump versions for `posthog-python@${{ steps.sampo-release.outputs.new_version }}`! " - emoji_reaction: "x" - # Build, publish, and tag each package whose version was bumped in this release. # A matrix over the publishable packages (N packages, like the posthog-ruby and # JS monorepos). The approval gate lives on `version-bump`, so this job has no @@ -436,16 +447,6 @@ jobs: "alertContext": "The package publish step failed." } - - name: Notify Slack - Failed - if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }} - uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main - with: - slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} - slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} - thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }} - message: "❌ Failed to publish `${{ matrix.package.name }}@${{ needs.version-bump.outputs.new_version }}`! " - emoji_reaction: "x" - # Side-effect-free rehearsal: runs ONLY when this workflow is manually # dispatched on a non-main branch (e.g. this PR branch). It applies the Sampo # version bump, builds openfeature-provider-posthog, and publishes it to diff --git a/RELEASING.md b/RELEASING.md index 568e9dd82..b2f7d9cd7 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -27,3 +27,13 @@ You can also trigger a release manually via the workflow's `workflow_dispatch` t > [!IMPORTANT] > Changesets must live in **`.sampo/changesets/`** (this is where `sampo add` puts them). Do **not** create them in the legacy `.changeset/` directory — Sampo and the `Release` workflow only read `.sampo/changesets/*.md`, so a changeset placed anywhere else is silently ignored and **no release is triggered**. + +## Failed or cancelled release + +The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and a publish job (`Publish posthog`, `Publish posthoganalytics`, or `Publish openfeature-provider-posthog`) was never created, for example when GitHub Actions stops a run before the publish jobs start. Once every publish job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository. + +Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed. + +Job inspection retries transient GitHub API failures up to three times. If inspection still fails, it logs the HTTP status or error type and message, then posts a fallback alert with the overall run status and logs link so the release can be checked manually. Error diagnostics stay in the run log, not in the Slack message. + +Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs new file mode 100644 index 000000000..8f5c9b265 --- /dev/null +++ b/scripts/release-failure-notification.cjs @@ -0,0 +1,103 @@ +const { readdirSync, readFileSync } = require('node:fs'); +const { join } = require('node:path'); + +const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']); +const publishJobs = ['Publish posthog', 'Publish posthoganalytics', 'Publish openfeature-provider-posthog']; + +const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`); + +function readThreadTimestamp(directory, attempt) { + if (!directory) return undefined; + try { + const entries = readdirSync(directory); + // A single matching artifact is extracted straight into the directory. + // Failed-job reruns reuse the approval thread from the latest earlier attempt. + const files = entries.includes('thread-ts.txt') + ? [join(directory, 'thread-ts.txt')] + : entries + .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1]) + .filter(value => value && Number(value) <= attempt) + .sort((a, b) => Number(b) - Number(a)) + .map(value => join(directory, `release-slack-thread-${value}`, 'thread-ts.txt')); + if (files.length === 0) return undefined; + const timestamp = readFileSync(files[0], 'utf8').trim(); + return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined; + } catch (error) { + if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message); + return undefined; + } +} + +module.exports = async function releaseFailureNotification({ github, context, channel, threadDirectory }) { + const run = context.payload.workflow_run; + const threadTs = readThreadTimestamp(threadDirectory, run.run_attempt); + const destination = { channel, ...(threadTs ? { thread_ts: threadTs } : {}) }; + let jobs; + try { + jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { + ...context.repo, + run_id: run.id, + attempt_number: run.run_attempt, + per_page: 100, + }); + } catch (error) { + console.error('Failed to inspect release jobs:', error.status ?? error.name, error.message); + return { + ...destination, + text: [ + `⚠️ Could not inspect release jobs for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`, + 'Please check this release manually; individual job results are unavailable.', + `<${run.html_url}|View release logs> (run: ${run.conclusion})`, + ].join('\n'), + unfurl_links: false, + unfurl_media: false, + }; + } + + if (jobs.length === 0 && run.conclusion === 'cancelled') { + return null; + } + + const failedJobs = jobs.filter(job => unsuccessful.has(job.conclusion)); + const versionBump = jobs.find(job => job.name === 'Bump versions and commit to main'); + if (run.conclusion === 'failure' && failedJobs.length === 1 + && versionBump?.conclusion === 'failure' && versionBump.steps?.length === 0 + && jobs.some(job => job.name === 'Notify Slack - Rejected' + && job.steps?.some(step => step.name === 'Notify Slack - Rejected' && step.conclusion === 'success'))) { + return null; + } + + const pypiPublished = publishJobs.every(name => { + const cells = jobs.filter(job => isJob(job, name)); + return cells.length > 0 && cells.every(job => job.conclusion === 'success'); + }); + const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released'); + const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`); + + if (run.conclusion !== 'success' && failedJobs.length === 0) { + for (const name of publishJobs) { + if (!jobs.some(job => isJob(job, name))) problems.push(`• ${name}: never ran`); + } + } + + if (pypiPublished && (!releasedNotification || releasedNotification.conclusion === 'skipped')) { + problems.push('• The released Slack notification did not run.'); + } + + if (problems.length === 0 && run.conclusion === 'success') { + return null; + } + + return { + ...destination, + text: [ + pypiPublished + ? `⚠️ PyPI packages published; follow-up problems for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).` + : `❌ Release process failed or was cancelled for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`, + ...problems, + `<${run.html_url}|View release logs> (run: ${run.conclusion})`, + ].join('\n'), + unfurl_links: false, + unfurl_media: false, + }; +};