ci: don't let a dead Depot runner cancel a merge-queue run

scripts/merge-queue-fail-fast.py

@@ -13,6 +13,13 @@
 failed and cancelled `needs`, and reports failure within seconds, and the
 queue evicts the entry and rebuilds the ones behind it.
 
+A job whose runner died is the exception (see `runner_lost`): it says
+nothing about the PR, and cancelling the run for it turned one dead runner
+into an eviction plus a burst of cancelled jobs. Those are left alone, so
+`ci-ok` decides when the run finishes. The job can't be re-run instead:
+`POST .../actions/jobs/{id}/rerun` is refused until the whole run has
+completed, and by then `ci-ok` has already reported.
+
 The script never fails its own job: any API error is a warning and the
 watch carries on (or gives up), leaving the run to finish as before.
 """
@@ -31,14 +38,32 @@
 # Job conclusions that make `ci-ok` fail. `cancelled` is left out: it
 # means someone (the queue, a janitor, a person) is already cancelling.
 FATAL = {"failure", "timed_out"}
+# Check-run annotations that mean the job's runner went away, not that a
+# step failed. On Depot runners these arrive as a job `failure` (its step
+# shows `cancelled`) about a minute into the job, whatever the job runs;
+# "lost communication" is the same death noticed by GitHub's 10-minute
+# heartbeat. Matched case-insensitively against failure-level messages.
+RUNNER_LOSS = ("lost communication", "shutdown signal", "step canceled by github")
 
 
 def fatal_jobs(jobs):
     """Names of the completed jobs whose conclusion dooms `ci-ok`."""
-    return sorted(
-        j["name"]
+    return sorted(j["name"] for j in failed_jobs(jobs))
+
+
+def failed_jobs(jobs):
+    return [
+        j
         for j in jobs
         if j.get("status") == "completed" and j.get("conclusion") in FATAL and j.get("name") != GATE
+    ]
+
+
+def runner_lost(annotations):
+    """True when a job's failure annotations say its runner died."""
+    return any(
+        a.get("annotation_level") == "failure" and any(m in (a.get("message") or "").lower() for m in RUNNER_LOSS)
+        for a in annotations
     )
 
 
@@ -74,6 +99,11 @@ def list_jobs(repo, run_id, token):
         page += 1
 
 
+def annotations(repo, job_id, token):
+    # A job's id is also its check-run id.
+    return request("GET", f"repos/{repo}/check-runs/{job_id}/annotations?per_page=100", token)
+
+
 def summary(text):
     print(text)
     path = os.environ.get("GITHUB_STEP_SUMMARY")
@@ -89,6 +119,7 @@ def main():
     interval = int(os.environ.get("POLL_SECONDS", "30"))
     deadline = time.monotonic() + int(os.environ.get("WATCH_MINUTES", "100")) * 60
     run = None
+    lost = {}  # job id -> runner_lost verdict; a completed job's annotations don't change
     while time.monotonic() < deadline:
         try:
             if run is None:
@@ -100,7 +131,15 @@ def main():
             if request("GET", f"repos/{repo}/actions/runs/{run['id']}", token)["status"] == "completed":
                 print("CI run completed; nothing to do")
                 return 0
-            failed = fatal_jobs(list_jobs(repo, run["id"], token))
+            failed = []
+            for j in failed_jobs(list_jobs(repo, run["id"], token)):
+                if j["id"] not in lost:
+                    lost[j["id"]] = runner_lost(annotations(repo, j["id"], token))
+                    if lost[j["id"]]:
+                        summary(f"Not cancelling for `{j['name']}`: its runner died ({j.get('html_url', '')}); `ci-ok` decides.")
+                if not lost[j["id"]]:
+                    failed.append(j["name"])
+            failed.sort()
             if failed:
                 request("POST", f"repos/{repo}/actions/runs/{run['id']}/cancel", token)
                 summary(

scripts/tests/test_merge_queue_fail_fast.py

@@ -1,9 +1,12 @@
 """Tests for scripts/merge-queue-fail-fast.py."""
 
 import importlib.util
+import io
 import re
 import unittest
+from contextlib import redirect_stdout
 from pathlib import Path
+from unittest import mock
 
 ROOT = Path(__file__).parents[2]
 CI = ROOT / ".github" / "workflows" / "ci.yml"
@@ -37,6 +40,73 @@ def test_cancelled_jobs_and_the_gate_itself_are_ignored(self):
         self.assertEqual(ff.fatal_jobs(jobs), [])
 
 
+def note(message, level="failure"):
+    return {"annotation_level": level, "message": message}
+
+
+class RunnerLost(unittest.TestCase):
+    """Annotations GitHub attached to jobs whose Depot runner died (2026-10-09)."""
+
+    def test_depot_runner_deaths_are_recognised(self):
+        for message in (
+            "Step canceled by GitHub (see: https://depot.dev/docs/github-actions/troubleshooting#error-step-canceled-by-github)",
+            "The runner has received a shutdown signal. This can happen when the runner service is stopped, "
+            "or a manually started runner is canceled.",
+            "The self-hosted runner lost communication with the server. Verify the machine is running and has "
+            "a healthy network connection.",
+        ):
+            with self.subTest(message=message[:30]):
+                self.assertTrue(ff.runner_lost([note("Node.js 20 is deprecated.", "warning"), note(message)]))
+
+    def test_a_real_failure_is_not_a_runner_loss(self):
+        self.assertFalse(ff.runner_lost([]))
+        self.assertFalse(ff.runner_lost([note("Process completed with exit code 101.")]))
+
+    def test_only_failure_level_annotations_count(self):
+        self.assertFalse(ff.runner_lost([note("Step canceled by GitHub", "warning")]))
+
+
+class Watch(unittest.TestCase):
+    """main() against a fake API: which failed jobs make it cancel the run."""
+
+    def watch(self, notes_by_job):
+        jobs = [
+            {"id": i, "name": f"job{i}", "status": "completed", "conclusion": "failure", "html_url": f"u{i}"}
+            for i in notes_by_job
+        ]
+        calls = []
+
+        def request(method, path, token):
+            calls.append((method, path))
+            if "/annotations" in path:
+                return notes_by_job[int(path.split("/")[-2])]
+            if path.endswith("/jobs?filter=latest&per_page=100&page=1"):
+                return {"jobs": jobs}
+            if "/workflows/ci.yml/runs" in path:
+                return {"workflow_runs": [{"id": 7, "html_url": "run"}]}
+            if path.endswith("/runs/7"):
+                # Report completion on the second poll so the watch ends.
+                done = sum(1 for _, p in calls if p.endswith("/runs/7")) > 1
+                return {"status": "completed" if done else "in_progress"}
+            return None
+
+        env = {"GITHUB_REPOSITORY": "o/r", "HEAD_SHA": "abc", "GH_TOKEN": "t", "POLL_SECONDS": "0"}
+        with mock.patch.dict(ff.os.environ, env, clear=True), mock.patch.object(ff, "request", request), \
+                mock.patch.object(ff.time, "sleep"), redirect_stdout(io.StringIO()):
+            ff.main()
+        return calls
+
+    def test_a_runner_loss_does_not_cancel_the_run(self):
+        calls = self.watch({1: [note("Step canceled by GitHub (see: https://depot.dev/...)")]})
+        self.assertNotIn(("POST", "repos/o/r/actions/runs/7/cancel"), calls)
+        # The verdict is cached: annotations are fetched once across polls.
+        self.assertEqual(sum("/annotations" in p for _, p in calls), 1)
+
+    def test_a_real_failure_still_cancels_the_run(self):
+        calls = self.watch({1: [note("The runner has received a shutdown signal.")], 2: [note("exit code 1")]})
+        self.assertIn(("POST", "repos/o/r/actions/runs/7/cancel"), calls)
+
+
 class CiWorkflowContract(unittest.TestCase):
     """Cancelling on a failed job is safe only while ci.yml keeps these."""