From 13b036c0c9cf0f3adfeb3f30dfe007aefc3b5e8c Mon Sep 17 00:00:00 2001 From: bjorn Date: Thu, 30 Jul 2026 07:42:41 +0200 Subject: [PATCH] ci: retry acceptance tests on CI to absorb runner flake The acceptance matrix ran with `fullyParallel: true` and no `retries`, so the Playwright default of 0 applied and any single transient failure took down a whole matrix row. That was happening regularly. Three runs in one four-minute window failed on three different specs, one of them on master, and every failure was a timeout rather than a failed assertion - twice specifically a `page.goto` that never finished loading a fixture page within the 30s default: master select-in-submenu.js:108 timeout after 35.0s fix/issue-809 nested-triggers-autohide.js:49 page.goto timeout fix/issue-812 menu-title-icon-alignment.js:70 page.goto timeout Specs that normally finish in 2-6s were taking 10-23s in those runs, so the runner was simply oversubscribed. Re-running the failed job passed 9/9 with no code change. Retry twice on CI only. Assertions stay exactly as strict, this just stops a loaded runner from being reported as a broken build. Locally retries would only mask a genuinely flaky spec, so they stay off there. --- playwright.config.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/playwright.config.js b/playwright.config.js index 307bd44f..af64a6a5 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -5,6 +5,13 @@ module.exports = defineConfig({ testDir: './test/specs', testMatch: '*.js', fullyParallel: true, + // Retry on CI only. The specs themselves are deterministic, but the runner is + // shared and oversubscribed, and a loaded one occasionally takes longer than + // the 30s default timeout just to load a fixture page - the failure is a + // `page.goto` timeout rather than a failed assertion. Without retries a single + // slow page load fails a whole matrix row. Locally, retries would only hide + // a genuinely flaky spec, so leave them off there. + retries: process.env.CI ? 2 : 0, reporter: 'list', use: { trace: 'retain-on-failure',