From d07371b302ef0eb82acc124d5b06c58650ca81cb Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Wed, 30 Sep 2026 21:23:13 +0200 Subject: [PATCH 1/2] Exit the oplog populator when the metastore stream is invalidated Dropping __metastore (as a D/R sink bootstrap does) closes the change stream for good. Since BB-879 these events are just skipped, so the process stays up but no longer sees bucket changes: buckets created or restored afterwards are never listened to, and lifecycle, replication and notifications silently stop for them. Exit on drop, rename, dropDatabase and invalidate instead, so that the restart goes through setup() again: re-list the buckets, open a new stream and stop listening to buckets that are gone. This is what the crash used to do before BB-879, now on purpose. Issue: BB-897 --- extensions/oplogPopulator/OplogPopulator.js | 12 ++++++++++ tests/unit/oplogPopulator/oplogPopulator.js | 25 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/extensions/oplogPopulator/OplogPopulator.js b/extensions/oplogPopulator/OplogPopulator.js index 045bb2203..78b735e75 100644 --- a/extensions/oplogPopulator/OplogPopulator.js +++ b/extensions/oplogPopulator/OplogPopulator.js @@ -199,6 +199,18 @@ class OplogPopulator { } break; } + case 'drop': + case 'rename': + case 'dropDatabase': + case 'invalidate': + // The change stream is closed for good, and buckets may have been removed or + // restored meanwhile: restart so that setup() re-syncs them on a new stream. + this._logger.error('Metastore change stream invalidated, exiting', { + method: 'OplogPopulator._handleChangeStreamChange', + type: change.operationType, + }); + process.exit(1); + return; default: this._logger.info('Skipping unsupported change stream event', { method: 'OplogPopulator._handleChangeStreamChange', diff --git a/tests/unit/oplogPopulator/oplogPopulator.js b/tests/unit/oplogPopulator/oplogPopulator.js index 2ee24a9d3..8a1ebed10 100644 --- a/tests/unit/oplogPopulator/oplogPopulator.js +++ b/tests/unit/oplogPopulator/oplogPopulator.js @@ -857,7 +857,7 @@ describe('OplogPopulator', () => { has: () => true, }; const changeDocument = { - operationType: 'drop', + operationType: 'modify', documentKey: { _id: 'example-bucket', }, @@ -878,9 +878,8 @@ describe('OplogPopulator', () => { has: () => true, }; const eventProcessed = sinon.stub(oplogPopulator._metricsHandler, 'onOplogEventProcessed'); - // "drop" and "invalidate" events carry no documentKey const changeDocument = { - operationType: 'drop', + operationType: 'createIndexes', }; await oplogPopulator._handleChangeStreamChangeEvent(changeDocument); assert(stopListening.notCalled); @@ -888,6 +887,26 @@ describe('OplogPopulator', () => { assert(eventProcessed.calledOnce); }); + ['drop', 'rename', 'dropDatabase', 'invalidate'].forEach(opType => { + it(`should exit when the change stream gets invalidated (${opType})`, async () => { + const exit = sinon.stub(process, 'exit'); + const stopListening = sinon.stub().resolves(); + const startListening = sinon.stub().resolves(); + oplogPopulator._allocator = { + stopListeningToBucket: stopListening, + listenToBucket: startListening, + has: () => true, + }; + const eventProcessed = sinon.stub(oplogPopulator._metricsHandler, 'onOplogEventProcessed'); + // these events carry no documentKey + await oplogPopulator._handleChangeStreamChangeEvent({ operationType: opType }); + assert(exit.calledOnceWith(1)); + assert(stopListening.notCalled); + assert(startListening.notCalled); + assert(eventProcessed.notCalled); + }); + }); + ['replace', 'insert', 'update'].forEach(opType => { it(`should stop listening if bucket becomes invalid (${opType})`, () => { const stopListening = sinon.stub().resolves(); From 073f2a1ba79baecc451b6dd4db45c327ac02e3e5 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Thu, 1 Oct 2026 14:42:05 +0200 Subject: [PATCH 2/2] Release 9.6.0-preview.3 Issue: BB-897 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e7f4a954f..4b9837fae 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@scality/backbeat", - "version": "9.6.0-preview.2", + "version": "9.6.0-preview.3", "description": "Asynchronous queue and job manager", "main": "index.js", "scripts": {