Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions .github/workflows/release-failure-notification.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
name: Release failure notification

on:
workflow_run:
workflows: [Release]
types: [completed]
branches: [main]

permissions:
actions: read
contents: read

jobs:
notify-failed:
name: Notify Slack - Failed
runs-on: ubuntu-latest
if: >-
github.event.workflow_run.head_repository.full_name == github.repository &&
(github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch')
timeout-minutes: 10
steps:
# This workflow must survive cancellation or runner startup failure in Release.
# Never execute code from the triggering run's head in this privileged workflow.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false

- name: Download approval thread timestamps
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
pattern: release-slack-thread-*
path: ${{ runner.temp }}/release-slack-threads
github-token: ${{ github.token }}
run-id: ${{ github.event.workflow_run.id }}

- name: Check release jobs
id: release-status
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SLACK_CHANNEL_ID: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
THREAD_DIRECTORY: ${{ runner.temp }}/release-slack-threads
with:
retries: 3
script: |
const notification = require('./scripts/release-failure-notification.cjs');
const payload = await notification({ github, context, channel: process.env.SLACK_CHANNEL_ID, threadDirectory: process.env.THREAD_DIRECTORY });
if (payload) core.setOutput('payload', JSON.stringify(payload));

- name: Notify Slack - Failed
if: steps.release-status.outputs.payload != ''
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
method: chat.postMessage
token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
errors: true
retries: 5
payload: ${{ steps.release-status.outputs.payload }}
41 changes: 21 additions & 20 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,27 @@ jobs:
slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
posthog_project_api_key: ${{ secrets.POSTHOG_PROJECT_API_KEY }}

preserve-slack-thread:
name: Preserve Slack thread
needs: notify-approval-needed
if: needs.notify-approval-needed.outputs.slack_ts != ''
runs-on: ubuntu-latest
timeout-minutes: 5
continue-on-error: true
permissions: {}
steps:
- name: Save approval thread timestamp
env:
THREAD_TS: ${{ needs.notify-approval-needed.outputs.slack_ts }}
run: printf '%s' "$THREAD_TS" > thread-ts.txt

- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-slack-thread-${{ github.run_attempt }}
path: thread-ts.txt
retention-days: 30
if-no-files-found: error

# Bump versions, commit to main, and regenerate references. This is the single
# approval-gated job (environment "Release"). It does NOT publish — publishing
# happens in the `publish` matrix job below, which has no environment so the
Expand Down Expand Up @@ -229,16 +250,6 @@ jobs:
"alertContext": "The version bump/changelog step failed before packages were published."
}

- name: Notify Slack - Failed
if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }}
uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main
with:
slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }}
message: "❌ Failed to bump versions for `posthog-python@${{ steps.sampo-release.outputs.new_version }}`! <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"
emoji_reaction: "x"

# Build, publish, and tag each package whose version was bumped in this release.
# A matrix over the publishable packages (N packages, like the posthog-ruby and
# JS monorepos). The approval gate lives on `version-bump`, so this job has no
Expand Down Expand Up @@ -436,16 +447,6 @@ jobs:
"alertContext": "The package publish step failed."
}

- name: Notify Slack - Failed
if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }}
uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main
with:
slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }}
message: "❌ Failed to publish `${{ matrix.package.name }}@${{ needs.version-bump.outputs.new_version }}`! <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"
emoji_reaction: "x"

# Side-effect-free rehearsal: runs ONLY when this workflow is manually
# dispatched on a non-main branch (e.g. this PR branch). It applies the Sampo
# version bump, builds openfeature-provider-posthog, and publishes it to
Expand Down
10 changes: 10 additions & 0 deletions RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,13 @@ You can also trigger a release manually via the workflow's `workflow_dispatch` t

> [!IMPORTANT]
> Changesets must live in **`.sampo/changesets/`** (this is where `sampo add` puts them). Do **not** create them in the legacy `.changeset/` directory — Sampo and the `Release` workflow only read `.sampo/changesets/*.md`, so a changeset placed anywhere else is silently ignored and **no release is triggered**.

## Failed or cancelled release

The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and a publish job (`Publish posthog`, `Publish posthoganalytics`, or `Publish openfeature-provider-posthog`) was never created, for example when GitHub Actions stops a run before the publish jobs start. Once every publish job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository.

Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed.

Job inspection retries transient GitHub API failures up to three times. If inspection still fails, it logs the HTTP status or error type and message, then posts a fallback alert with the overall run status and logs link so the release can be checked manually. Error diagnostics stay in the run log, not in the Slack message.

Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run.
103 changes: 103 additions & 0 deletions scripts/release-failure-notification.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
const { readdirSync, readFileSync } = require('node:fs');
const { join } = require('node:path');

const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']);
const publishJobs = ['Publish posthog', 'Publish posthoganalytics', 'Publish openfeature-provider-posthog'];

const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`);

function readThreadTimestamp(directory, attempt) {
if (!directory) return undefined;
try {
const entries = readdirSync(directory);
// A single matching artifact is extracted straight into the directory.
// Failed-job reruns reuse the approval thread from the latest earlier attempt.
const files = entries.includes('thread-ts.txt')
? [join(directory, 'thread-ts.txt')]
: entries
.map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1])
.filter(value => value && Number(value) <= attempt)
.sort((a, b) => Number(b) - Number(a))
.map(value => join(directory, `release-slack-thread-${value}`, 'thread-ts.txt'));
if (files.length === 0) return undefined;
const timestamp = readFileSync(files[0], 'utf8').trim();
return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined;
} catch (error) {
if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message);
return undefined;
}
}

module.exports = async function releaseFailureNotification({ github, context, channel, threadDirectory }) {
const run = context.payload.workflow_run;
const threadTs = readThreadTimestamp(threadDirectory, run.run_attempt);
const destination = { channel, ...(threadTs ? { thread_ts: threadTs } : {}) };
let jobs;
try {
jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, {
...context.repo,
run_id: run.id,
attempt_number: run.run_attempt,
per_page: 100,
});
} catch (error) {
console.error('Failed to inspect release jobs:', error.status ?? error.name, error.message);
return {
...destination,
text: [
`⚠️ Could not inspect release jobs for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`,
'Please check this release manually; individual job results are unavailable.',
`<${run.html_url}|View release logs> (run: ${run.conclusion})`,
].join('\n'),
unfurl_links: false,
unfurl_media: false,
};
}

if (jobs.length === 0 && run.conclusion === 'cancelled') {
return null;
}

const failedJobs = jobs.filter(job => unsuccessful.has(job.conclusion));
const versionBump = jobs.find(job => job.name === 'Bump versions and commit to main');
if (run.conclusion === 'failure' && failedJobs.length === 1
&& versionBump?.conclusion === 'failure' && versionBump.steps?.length === 0
&& jobs.some(job => job.name === 'Notify Slack - Rejected'
&& job.steps?.some(step => step.name === 'Notify Slack - Rejected' && step.conclusion === 'success'))) {
return null;
}

const pypiPublished = publishJobs.every(name => {
const cells = jobs.filter(job => isJob(job, name));
return cells.length > 0 && cells.every(job => job.conclusion === 'success');
});
const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released');
const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`);

if (run.conclusion !== 'success' && failedJobs.length === 0) {
for (const name of publishJobs) {
if (!jobs.some(job => isJob(job, name))) problems.push(`• ${name}: never ran`);
}
}

if (pypiPublished && (!releasedNotification || releasedNotification.conclusion === 'skipped')) {
problems.push('• The released Slack notification did not run.');
}

if (problems.length === 0 && run.conclusion === 'success') {
return null;
}

return {
...destination,
text: [
pypiPublished
? `⚠️ PyPI packages published; follow-up problems for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`
: `❌ Release process failed or was cancelled for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`,
...problems,
`<${run.html_url}|View release logs> (run: ${run.conclusion})`,
].join('\n'),
unfurl_links: false,
unfurl_media: false,
};
};
Loading