From 9311b9e6a4e474193dfc257b8c9a7ff162513cea Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 31 Oct 2026 12:51:22 -0400 Subject: [PATCH 1/3] test(redis): a client keeps publishing after the cache loses its scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1002 reported a relay that answered HTTP 200 and never acknowledged another event once its cache restarted: Lua scripts live in Redis memory, so every EVALSHA after the restart answered NOSCRIPT. The adapter retries with a fresh scriptLoad now, but nothing proved it — the unit test stubs evalSha, and the production failure was in the workers that call it. The scenario flushes the script cache between two publishes with a message rate limit configured, because both limiters return early on an empty list and a scenario without one would pass whether or not the fallback existed. --- .../redis-script-cache.feature | 18 ++++++++++ .../redis-script-cache.feature.ts | 35 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 test/integration/features/redis-script-cache/redis-script-cache.feature create mode 100644 test/integration/features/redis-script-cache/redis-script-cache.feature.ts diff --git a/test/integration/features/redis-script-cache/redis-script-cache.feature b/test/integration/features/redis-script-cache/redis-script-cache.feature new file mode 100644 index 00000000..35adf664 --- /dev/null +++ b/test/integration/features/redis-script-cache/redis-script-cache.feature @@ -0,0 +1,18 @@ +Feature: The relay recovers when the cache forgets its Lua scripts + + A Redis restart empties the script cache, so every subsequent `EVALSHA` answers + `NOSCRIPT`. The relay calls `EVALSHA` for its rate limiters and its moderation + counters, which means the cache losing its scripts used to leave the relay + serving nothing at all: sockets stayed open, `HTTP` answered, and no event was + ever acknowledged again until the process was restarted (#1002). + + `SCRIPT FLUSH` reproduces that state exactly, without restarting a container and + without waiting for a reconnect. + + @script-cache + Scenario: A client publishes before and after the cache drops its scripts + Given someone called Alice + When Alice sends a text_note event with content "before the flush" + And the cache forgets its stored Lua scripts + And Alice sends a text_note event with content "after the flush" + And Alice sends a text_note event with content "and once more" diff --git a/test/integration/features/redis-script-cache/redis-script-cache.feature.ts b/test/integration/features/redis-script-cache/redis-script-cache.feature.ts new file mode 100644 index 00000000..5e5916d9 --- /dev/null +++ b/test/integration/features/redis-script-cache/redis-script-cache.feature.ts @@ -0,0 +1,35 @@ +import { Before, When } from '@cucumber/cucumber' +import { assocPath, pipe } from 'ramda' +import { createClient } from 'redis' + +import { getCacheConfig } from '../../../../src/cache/client' +import { SettingsStatic } from '../../../../src/utils/settings' + +// The relay only touches Lua when a rate limit is configured: both limiters +// return early on an empty list. A scenario that never configures one would pass +// whether or not the EVALSHA fallback exists, which is the definition of a test +// that proves nothing. A generous limit keeps the window from tripping while +// still forcing the script through `EVALSHA`. +Before({ tags: '@script-cache' }, async function () { + SettingsStatic._settings = pipe( + assocPath(['limits', 'rateLimiter', 'strategy'], 'ewma'), + assocPath(['limits', 'message', 'rateLimits'], [{ period: 60000, rate: 100 }]), + assocPath(['limits', 'message', 'ipWhitelist'], []), + )(SettingsStatic._settings) as any +}) + +// `SCRIPT FLUSH` is the whole of a Redis restart as far as the relay can tell: +// the script cache is memory-resident and empty, so the next EVALSHA answers +// NOSCRIPT. Doing it this way keeps the scenario deterministic — no container +// to restart, no reconnect to wait for, and it exercises the adapter's retry +// rather than the client's reconnect logic. +When(/^the cache forgets its stored Lua scripts$/, async function () { + const client = createClient(getCacheConfig()) + await client.connect() + + try { + await client.sendCommand(['SCRIPT', 'FLUSH']) + } finally { + await client.quit() + } +}) From 80b0a9e52412ce4f554065fe5a6c6fac62af99b6 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 31 Oct 2026 12:53:33 -0400 Subject: [PATCH 2/3] chore(changeset): record the script-cache recovery coverage --- .changeset/redis-script-cache-recovery.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changeset/redis-script-cache-recovery.md diff --git a/.changeset/redis-script-cache-recovery.md b/.changeset/redis-script-cache-recovery.md new file mode 100644 index 00000000..256829d3 --- /dev/null +++ b/.changeset/redis-script-cache-recovery.md @@ -0,0 +1,9 @@ +--- +"nostream": patch +--- + +test(redis): prove the relay keeps publishing after the cache loses its scripts + +Nothing is shipped in this change — it is coverage for #1002. The relay issues `EVALSHA`, and Lua scripts live in the cache's memory, so a Redis restart leaves every script call answering `NOSCRIPT`. The adapter's retry for that was added in #920 and had only a stubbed unit test: `evalSha` was mocked, which cannot catch a retry that stops working through the workers that actually call it. + +The new integration scenario publishes, runs `SCRIPT FLUSH` against the same Redis, then publishes twice more. `SCRIPT FLUSH` is the whole of a restart as far as the relay can tell, and it is deterministic — no container to restart and no reconnect to wait for. The scenario configures a message rate limit on purpose, because both limiters return early on an empty list and a scenario without one would pass whether or not the fallback existed. From 4086a82ebdf6874e902111a242f970426096519c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 31 Oct 2026 13:28:38 -0400 Subject: [PATCH 3/3] fix(test): restore settings after the script-cache scenario The @script-cache hook rewrites SettingsStatic._settings so the message limiter actually runs, but one worker serves every scenario in the process and controllers read that object through createSettings() on each request. The mutation therefore outlived its scenario, and every scenario after it ran with a 100/min message limit, an empty ipWhitelist and the ewma strategy. Save the previous object in the tagged Before and put it back in a tagged After, which also runs when the scenario fails. --- .../redis-script-cache.feature.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/integration/features/redis-script-cache/redis-script-cache.feature.ts b/test/integration/features/redis-script-cache/redis-script-cache.feature.ts index 5e5916d9..ba6473e8 100644 --- a/test/integration/features/redis-script-cache/redis-script-cache.feature.ts +++ b/test/integration/features/redis-script-cache/redis-script-cache.feature.ts @@ -1,16 +1,24 @@ -import { Before, When } from '@cucumber/cucumber' +import { After, Before, When } from '@cucumber/cucumber' import { assocPath, pipe } from 'ramda' import { createClient } from 'redis' +import { Settings } from '../../../../src/@types/settings' import { getCacheConfig } from '../../../../src/cache/client' import { SettingsStatic } from '../../../../src/utils/settings' +// One worker serves every scenario in this process, and controllers read +// `SettingsStatic._settings` through `createSettings()` on each request, so a +// mutation here would outlive the scenario and make whatever runs next depend on +// test order. The tagged After below puts the previous object back. +let settingsBefore: Settings | undefined + // The relay only touches Lua when a rate limit is configured: both limiters // return early on an empty list. A scenario that never configures one would pass // whether or not the EVALSHA fallback exists, which is the definition of a test // that proves nothing. A generous limit keeps the window from tripping while // still forcing the script through `EVALSHA`. Before({ tags: '@script-cache' }, async function () { + settingsBefore = SettingsStatic._settings SettingsStatic._settings = pipe( assocPath(['limits', 'rateLimiter', 'strategy'], 'ewma'), assocPath(['limits', 'message', 'rateLimits'], [{ period: 60000, rate: 100 }]), @@ -18,6 +26,10 @@ Before({ tags: '@script-cache' }, async function () { )(SettingsStatic._settings) as any }) +After({ tags: '@script-cache' }, async function () { + SettingsStatic._settings = settingsBefore +}) + // `SCRIPT FLUSH` is the whole of a Redis restart as far as the relay can tell: // the script cache is memory-resident and empty, so the next EVALSHA answers // NOSCRIPT. Doing it this way keeps the scenario deterministic — no container