diff --git a/.changeset/redis-script-cache-recovery.md b/.changeset/redis-script-cache-recovery.md new file mode 100644 index 00000000..256829d3 --- /dev/null +++ b/.changeset/redis-script-cache-recovery.md @@ -0,0 +1,9 @@ +--- +"nostream": patch +--- + +test(redis): prove the relay keeps publishing after the cache loses its scripts + +Nothing is shipped in this change — it is coverage for #1002. The relay issues `EVALSHA`, and Lua scripts live in the cache's memory, so a Redis restart leaves every script call answering `NOSCRIPT`. The adapter's retry for that was added in #920 and had only a stubbed unit test: `evalSha` was mocked, which cannot catch a retry that stops working through the workers that actually call it. + +The new integration scenario publishes, runs `SCRIPT FLUSH` against the same Redis, then publishes twice more. `SCRIPT FLUSH` is the whole of a restart as far as the relay can tell, and it is deterministic — no container to restart and no reconnect to wait for. The scenario configures a message rate limit on purpose, because both limiters return early on an empty list and a scenario without one would pass whether or not the fallback existed. diff --git a/test/integration/features/redis-script-cache/redis-script-cache.feature b/test/integration/features/redis-script-cache/redis-script-cache.feature new file mode 100644 index 00000000..35adf664 --- /dev/null +++ b/test/integration/features/redis-script-cache/redis-script-cache.feature @@ -0,0 +1,18 @@ +Feature: The relay recovers when the cache forgets its Lua scripts + + A Redis restart empties the script cache, so every subsequent `EVALSHA` answers + `NOSCRIPT`. The relay calls `EVALSHA` for its rate limiters and its moderation + counters, which means the cache losing its scripts used to leave the relay + serving nothing at all: sockets stayed open, `HTTP` answered, and no event was + ever acknowledged again until the process was restarted (#1002). + + `SCRIPT FLUSH` reproduces that state exactly, without restarting a container and + without waiting for a reconnect. + + @script-cache + Scenario: A client publishes before and after the cache drops its scripts + Given someone called Alice + When Alice sends a text_note event with content "before the flush" + And the cache forgets its stored Lua scripts + And Alice sends a text_note event with content "after the flush" + And Alice sends a text_note event with content "and once more" diff --git a/test/integration/features/redis-script-cache/redis-script-cache.feature.ts b/test/integration/features/redis-script-cache/redis-script-cache.feature.ts new file mode 100644 index 00000000..ba6473e8 --- /dev/null +++ b/test/integration/features/redis-script-cache/redis-script-cache.feature.ts @@ -0,0 +1,47 @@ +import { After, Before, When } from '@cucumber/cucumber' +import { assocPath, pipe } from 'ramda' +import { createClient } from 'redis' + +import { Settings } from '../../../../src/@types/settings' +import { getCacheConfig } from '../../../../src/cache/client' +import { SettingsStatic } from '../../../../src/utils/settings' + +// One worker serves every scenario in this process, and controllers read +// `SettingsStatic._settings` through `createSettings()` on each request, so a +// mutation here would outlive the scenario and make whatever runs next depend on +// test order. The tagged After below puts the previous object back. +let settingsBefore: Settings | undefined + +// The relay only touches Lua when a rate limit is configured: both limiters +// return early on an empty list. A scenario that never configures one would pass +// whether or not the EVALSHA fallback exists, which is the definition of a test +// that proves nothing. A generous limit keeps the window from tripping while +// still forcing the script through `EVALSHA`. +Before({ tags: '@script-cache' }, async function () { + settingsBefore = SettingsStatic._settings + SettingsStatic._settings = pipe( + assocPath(['limits', 'rateLimiter', 'strategy'], 'ewma'), + assocPath(['limits', 'message', 'rateLimits'], [{ period: 60000, rate: 100 }]), + assocPath(['limits', 'message', 'ipWhitelist'], []), + )(SettingsStatic._settings) as any +}) + +After({ tags: '@script-cache' }, async function () { + SettingsStatic._settings = settingsBefore +}) + +// `SCRIPT FLUSH` is the whole of a Redis restart as far as the relay can tell: +// the script cache is memory-resident and empty, so the next EVALSHA answers +// NOSCRIPT. Doing it this way keeps the scenario deterministic — no container +// to restart, no reconnect to wait for, and it exercises the adapter's retry +// rather than the client's reconnect logic. +When(/^the cache forgets its stored Lua scripts$/, async function () { + const client = createClient(getCacheConfig()) + await client.connect() + + try { + await client.sendCommand(['SCRIPT', 'FLUSH']) + } finally { + await client.quit() + } +})