From 1a200dc7006716feb7288fff33e569d6f9b98918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 10:52:21 -0500 Subject: [PATCH 01/10] Keep hub final blocks until they are merged A merger more than keepFinalBlocks behind LIB left a range of blocks that neither the merged-blocks files nor the hub could serve, so streams reading through the files waited there for the merger. --- CHANGELOG.md | 2 + forkable/forkable.go | 25 +++- forkable/options.go | 12 ++ hub/hub.go | 35 ++++++ hub/merged_blocks.go | 241 ++++++++++++++++++++++++++++++++++++++ hub/merged_blocks_test.go | 196 +++++++++++++++++++++++++++++++ 6 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 hub/merged_blocks.go create mode 100644 hub/merged_blocks_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 588dcad..512254f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add `hub.WithMergedBlocksStore`: the hub keeps its final blocks until the merged-blocks file holding them exists, so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMaxKeptFinalBlocks`, `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. +- Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `ForkableHub.WithoutPartials` and `stream.WithoutPartialBlocks`: stream from a hub fed with partial blocks without getting them, the last partial of each block arriving as the complete block, as for a relayer client that does not ask for partials. Lets an app that does not handle partial blocks share its hub with one that does. diff --git a/forkable/forkable.go b/forkable/forkable.go index a3cbbe4..d08d5cb 100644 --- a/forkable/forkable.go +++ b/forkable/forkable.go @@ -44,6 +44,10 @@ type Forkable struct { holdBlocksUntilLIB bool // if true, never passthrough anything before a LIB is set keptFinalBlocks int // how many blocks we keep behind LIB + // finalBlocksPurgeLimit, when set, can lower the block number under which final + // blocks are dropped, see WithFinalBlocksPurgeLimit. + finalBlocksPurgeLimit func(libNum, cutoff uint64) uint64 + includeInitialLIB bool failOnUnlinkableBlocksCount int @@ -860,7 +864,7 @@ func (p *Forkable) ProcessBlock(blk *pbbstream.Block, obj any) error { p.forkDB.MoveLIB(libRef) headBlockNum := newHeadBlock.Num() - _ = p.forkDB.PurgeBeforeLIBAndPartials(p.keptFinalBlocks, &headBlockNum) + _ = p.forkDB.PurgeBeforeLIBAndPartials(p.finalBlocksToKeep(libRef.Num()), &headBlockNum) if err := p.processIrreversibleSegment(irreversibleSegment, ppBlk.Block.AsRef()); err != nil { return err @@ -873,6 +877,25 @@ func (p *Forkable) ProcessBlock(blk *pbbstream.Block, obj any) error { return nil } +// finalBlocksToKeep is how many blocks are kept below a LIB at libNum: keptFinalBlocks, +// or more when finalBlocksPurgeLimit holds the purge back. +func (p *Forkable) finalBlocksToKeep(libNum uint64) int { + if p.finalBlocksPurgeLimit == nil { + return p.keptFinalBlocks + } + + var cutoff uint64 + if libNum > uint64(p.keptFinalBlocks) { + cutoff = libNum - uint64(p.keptFinalBlocks) + } + + limit := p.finalBlocksPurgeLimit(libNum, cutoff) + if limit >= cutoff { + return p.keptFinalBlocks + } + return int(libNum - limit) +} + func ids(blocks []*ForkableBlock) (ids []string) { ids = make([]string, len(blocks)) for i, obj := range blocks { diff --git a/forkable/options.go b/forkable/options.go index 6d7dacf..96dd837 100644 --- a/forkable/options.go +++ b/forkable/options.go @@ -104,6 +104,18 @@ func WithKeptFinalBlocks(count int) Option { } } +// WithFinalBlocksPurgeLimit lets the caller keep more final blocks than WithKeptFinalBlocks +// asks for. Each time the LIB moves, limit is called with the new LIB number and the +// cutoff the forkable is about to apply (blocks below it are dropped); returning a lower +// number keeps the blocks from that number on. A higher number is ignored. +// +// limit is called with the forkable locked, on the path of every block: it must not block. +func WithFinalBlocksPurgeLimit(limit func(libNum, cutoff uint64) uint64) Option { + return func(f *Forkable) { + f.finalBlocksPurgeLimit = limit + } +} + func EnsureBlockFlows(blockRef bstream.BlockRef) Option { return func(f *Forkable) { f.ensureBlockFlows = blockRef diff --git a/hub/hub.go b/hub/hub.go index e22ffb8..648c047 100644 --- a/hub/hub.go +++ b/hub/hub.go @@ -59,6 +59,10 @@ type ForkableHub struct { oneBlockDownloadConcurrency int + mergedBlocks *mergedBlocksTracker // nil unless WithMergedBlocksStore is used + maxKeptFinalBlocks int + mergedBlocksLagMetric forkable.Uint64Metric + lastProgressLog time.Time } @@ -105,6 +109,15 @@ func newForkableHub(liveSourceFactory bstream.SourceFactory, keepFinalBlocks int forkable.WithFilters(bstream.StepsAllWithPartial), }, extraForkableOptions...) + if hub.mergedBlocks != nil { + if hub.maxKeptFinalBlocks == 0 { + hub.maxKeptFinalBlocks = defaultMaxKeptFinalBlocksFactor * keepFinalBlocks + } + hub.mergedBlocks.bundleSize = bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) + hub.mergedBlocks.maxKeptFinalBlocks = uint64(max(hub.maxKeptFinalBlocks, keepFinalBlocks)) + forkableOptions = append(forkableOptions, forkable.WithFinalBlocksPurgeLimit(hub.mergedBlocks.purgeLimit)) + } + hub.forkableOptions = forkableOptions hub.forkable = forkable.New(bstream.HandlerFunc(hub.broadcastBlock), forkableOptions...) @@ -408,6 +421,14 @@ func (h *ForkableHub) bootstrapOnce(ctx context.Context) error { return fmt.Errorf("parsing filename: %w", err) } lowestBlockNum := substractAndRoundDownBlocks(refLibNum, uint64(h.keepFinalBlocks), bstream.DefaultMergedBlocksBundleSize) + if h.mergedBlocks != nil { + // Merged blocks that are behind move the starting point down to their last file. + if located, err := h.mergedBlocks.locate(ctx, lowestBlockNum, refLibNum); err != nil { + h.logger.Warn("cannot find the last merged-blocks file, bootstrapping as if merged blocks were up to date", zap.Error(err)) + } else { + lowestBlockNum = located + } + } filenames, err := h.unknownOneBlockFiles(sortedOneBlocksFiles, lowestBlockNum) if err != nil { @@ -452,6 +473,10 @@ func (h *ForkableHub) Run() { close(h.Ready) } + if h.mergedBlocks != nil { + go h.trackMergedBlocks() + } + liveSource.Run() } @@ -531,6 +556,16 @@ func (h *ForkableHub) linkLiveUsingOneBlocks(ctx context.Context, blk *pbbstream lastKnownLib := h.forkable.LowestBlockNum() if !h.forkable.ForkDBHasLib() { lastKnownLib = blk.LibNum + if h.mergedBlocks != nil { + // Same starting point as bootstrap: the last merged-blocks file, so the hub + // covers every block the files do not hold yet. + lowestBlockNum := substractAndRoundDownBlocks(blk.LibNum, uint64(h.keepFinalBlocks), bstream.DefaultMergedBlocksBundleSize) + if located, err := h.mergedBlocks.locate(ctx, lowestBlockNum, blk.LibNum); err != nil { + h.logger.Warn("cannot find the last merged-blocks file, linking from the live block's LIB", zap.Error(err)) + } else { + lastKnownLib = min(lastKnownLib, located) + } + } } h.logger.Info("linking live block using one blocks", zap.Uint64("processed_block", blk.Number), zap.Uint64("last_know_lib", lastKnownLib)) diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go new file mode 100644 index 0000000..dbf4230 --- /dev/null +++ b/hub/merged_blocks.go @@ -0,0 +1,241 @@ +package hub + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/streamingfast/bstream" + "github.com/streamingfast/bstream/forkable" + "github.com/streamingfast/dstore" + "go.uber.org/zap" +) + +// defaultMaxKeptFinalBlocksFactor sets the default limit on the final blocks a hub keeps +// while merged blocks are behind, as a multiple of its keepFinalBlocks. +const defaultMaxKeptFinalBlocksFactor = 10 + +// mergedBlocksPollInterval is how often the hub checks whether the next merged-blocks file +// showed up. It is a var so tests can shrink it. +var mergedBlocksPollInterval = 5 * time.Second + +// mergedBlocksBehindLogInterval is how often the hub warns that it keeps extra final +// blocks because merged blocks are behind. +const mergedBlocksBehindLogInterval = 30 * time.Second + +// WithMergedBlocksStore makes the hub keep its final blocks until they are available in a +// merged-blocks file of store, instead of dropping everything older than keepFinalBlocks +// below LIB. Without it, a merger falling more than keepFinalBlocks behind LIB leaves a +// range of blocks that neither the merged-blocks files nor the hub can serve, and every +// stream reading through the files waits there until the merger catches up. +// +// The hub learns about new files with one FileExists call on the next expected file, at +// most every 5 seconds and only once LIB passed the end of that file. +// +// The extra blocks are held in memory, up to WithMaxKeptFinalBlocks. +func WithMergedBlocksStore(store dstore.Store) Option { + return func(h *ForkableHub) { + h.mergedBlocks = &mergedBlocksTracker{store: store} + } +} + +// WithMaxKeptFinalBlocks limits how many final blocks the hub keeps below LIB while it +// waits for merged blocks, see WithMergedBlocksStore. Past that limit the oldest blocks +// are dropped even though no merged-blocks file holds them. Defaults to 10 times the +// hub's keepFinalBlocks; a value below keepFinalBlocks is raised to it. +func WithMaxKeptFinalBlocks(count int) Option { + return func(h *ForkableHub) { + h.maxKeptFinalBlocks = count + } +} + +// WithMergedBlocksLagMetric reports how many blocks LIB is ahead of the last block found +// in the merged-blocks store. It needs WithMergedBlocksStore. With a merger keeping up, +// the value moves between 1 and a little over one merged-blocks file; it keeps growing +// when the merger is stopped or too slow. +func WithMergedBlocksLagMetric(metric forkable.Uint64Metric) Option { + return func(h *ForkableHub) { + h.mergedBlocksLagMetric = metric + } +} + +// mergedBlocksTracker follows the merged-blocks store from the hub's point of view: which +// is the first file, going up, that is not there yet. +type mergedBlocksTracker struct { + store dstore.Store + bundleSize uint64 + maxKeptFinalBlocks uint64 + + lookupLock sync.Mutex // serializes the store lookups + + // nextBase is the base block number of the first merged-blocks file not seen in the + // store. Every file below it was seen, or sits under where the hub started looking. + nextBase atomic.Uint64 + known atomic.Bool +} + +func mergedBlocksFilename(base uint64) string { + return fmt.Sprintf("%010d", base) +} + +// fileExists tells whether the merged-blocks file starting at base is in the store. A +// lookup that fails is retried like a bootstrap attempt is: bootstrapMaxAttempts times, +// bootstrapRetryDelay apart. A missing file is an answer, not a failure. +func (t *mergedBlocksTracker) fileExists(ctx context.Context, base uint64) (exists bool, err error) { + for attempt := range bootstrapMaxAttempts { + if attempt > 0 { + time.Sleep(bootstrapRetryDelay) + } + if exists, err = t.store.FileExists(ctx, mergedBlocksFilename(base)); err == nil { + return exists, nil + } + } + return false, fmt.Errorf("looking up merged-blocks file %s: %w", mergedBlocksFilename(base), err) +} + +// locate finds where the merged blocks end, looking down from the file that holds +// lowestBlockNum, and returns the block the hub must start from for its blocks to overlap +// the last merged-blocks file. It does not go further than maxKeptFinalBlocks below libNum. +func (t *mergedBlocksTracker) locate(ctx context.Context, lowestBlockNum, libNum uint64) (uint64, error) { + t.lookupLock.Lock() + defer t.lookupLock.Unlock() + + firstBase := bstream.GetProtocolFirstStreamableBlock / t.bundleSize * t.bundleSize + base := lowestBlockNum / t.bundleSize * t.bundleSize + + for { + exists, err := t.fileExists(ctx, base) + if err != nil { + return 0, err + } + if exists { + t.nextBase.Store(base + t.bundleSize) + break + } + if base <= firstBase || libNum-min(libNum, base)+t.bundleSize > t.maxKeptFinalBlocks { + t.nextBase.Store(base) + break + } + base -= t.bundleSize + } + t.known.Store(true) + + return max(min(base, lowestBlockNum), bstream.GetProtocolFirstStreamableBlock), nil +} + +// advance moves nextBase over the files that showed up. A file is only looked up once LIB +// passed its last block, since the merger cannot have written it before. +func (t *mergedBlocksTracker) advance(ctx context.Context, libNum uint64) error { + t.lookupLock.Lock() + defer t.lookupLock.Unlock() + + for { + base := t.nextBase.Load() + if libNum < base+t.bundleSize { + return nil + } + + exists, err := t.fileExists(ctx, base) + if err != nil { + return err + } + if !exists { + return nil + } + t.nextBase.Store(base + t.bundleSize) + } +} + +// lag is how many blocks libNum is ahead of the last merged block. +func (t *mergedBlocksTracker) lag(libNum uint64) uint64 { + nextBase := t.nextBase.Load() + if libNum < nextBase { + return 0 + } + return libNum + 1 - nextBase +} + +// purgeLimit is a forkable.WithFinalBlocksPurgeLimit function. It keeps the blocks of the +// last merged-blocks file and everything above: a stream reading the files joins the hub +// on a block it received from a file, so the hub has to overlap the last one. +func (t *mergedBlocksTracker) purgeLimit(libNum, cutoff uint64) uint64 { + if !t.known.Load() { + return cutoff + } + + var limit uint64 + if nextBase := t.nextBase.Load(); nextBase > t.bundleSize { + limit = nextBase - t.bundleSize + } + if libNum > t.maxKeptFinalBlocks { + limit = max(limit, libNum-t.maxKeptFinalBlocks) + } + return min(limit, cutoff) +} + +// MergedBlocksUpTo returns the first block number the hub has not found in a merged-blocks +// file: every block below it can be read from the merged-blocks store. ok is false when +// the hub does not follow that store (see WithMergedBlocksStore) or has not looked yet. +func (h *ForkableHub) MergedBlocksUpTo() (blockNum uint64, ok bool) { + if h == nil || h.mergedBlocks == nil || !h.mergedBlocks.known.Load() { + return 0, false + } + return h.mergedBlocks.nextBase.Load(), true +} + +// trackMergedBlocks keeps the merged blocks tracker and the lag metric up to date until +// the hub terminates. +func (h *ForkableHub) trackMergedBlocks() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ticker := time.NewTicker(mergedBlocksPollInterval) + defer ticker.Stop() + + var lastBehindLog time.Time + for { + select { + case <-h.Terminating(): + return + case <-ticker.C: + } + + if !h.IsReady() { + continue + } + + t := h.mergedBlocks + libNum := h.forkable.LIBNum() + + if !t.known.Load() { + if _, err := t.locate(ctx, h.forkable.LowestBlockNum(), libNum); err != nil { + h.logger.Warn("cannot find the last merged-blocks file", zap.Error(err)) + continue + } + } + + if err := t.advance(ctx, libNum); err != nil { + h.logger.Warn("cannot check for new merged-blocks files", zap.Error(err)) + } + + lag := t.lag(libNum) + if h.mergedBlocksLagMetric != nil { + h.mergedBlocksLagMetric.SetUint64(lag) + } + + if lag > uint64(h.keepFinalBlocks) && time.Since(lastBehindLog) >= mergedBlocksBehindLogInterval { + lastBehindLog = time.Now() + lowest := h.forkable.LowestBlockNum() + h.logger.Warn("merged blocks are behind, keeping more final blocks in memory to cover for them", + zap.Uint64("lib_num", libNum), + zap.Uint64("next_merged_blocks_file", t.nextBase.Load()), + zap.Uint64("merged_blocks_lag", lag), + zap.Uint64("lowest_block_num", lowest), + zap.Uint64("max_kept_final_blocks", t.maxKeptFinalBlocks), + zap.Bool("blocks_missing_between_merged_and_hub", lowest > t.nextBase.Load()), + ) + } + } +} diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go new file mode 100644 index 0000000..2d31988 --- /dev/null +++ b/hub/merged_blocks_test.go @@ -0,0 +1,196 @@ +package hub + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/streamingfast/bstream" + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + "github.com/streamingfast/dstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testChain returns blocks from..to, each one final two blocks later. +func testChain(from, to uint64) (out []*pbbstream.Block) { + id := func(num uint64) string { return fmt.Sprintf("%08x", num) } + for num := from; num <= to; num++ { + out = append(out, bstream.TestBlockWithLIBNum(id(num), id(num-1), num-2)) + } + return out +} + +func setMergedBlocksFiles(store *dstore.MockStore, bases ...uint64) { + for _, base := range bases { + store.SetFile(mergedBlocksFilename(base), []byte("merged")) + } +} + +func newMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBases []uint64, opts ...Option) (*ForkableHub, *dstore.MockStore) { + t.Helper() + + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + oneBlocksStore := dstore.NewMockStore(nil) + AddToMockStore(t, oneBlocksStore, oneBlocks...) + + mergedBlocksStore := dstore.NewMockStore(nil) + setMergedBlocksFiles(mergedBlocksStore, mergedBases...) + + lsf := bstream.NewTestSourceFactory() + fh := NewForkableHubWithOptions(lsf.NewSource, 200, oneBlocksStore, append([]Option{WithMergedBlocksStore(mergedBlocksStore)}, opts...)) + return fh, mergedBlocksStore +} + +func processBlocks(t *testing.T, fh *ForkableHub, blocks []*pbbstream.Block) { + t.Helper() + for _, blk := range blocks { + require.NoError(t, fh.ProcessBlock(blk, nil)) + } +} + +func TestForkableHub_KeepsFinalBlocksUntilMerged(t *testing.T) { + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{100, 200, 300}) + require.NoError(t, fh.bootstrap()) + require.Equal(t, uint64(300), fh.forkable.LowestBlockNum()) + + // LIB reaches 898 while the last merged-blocks file still ends at 399 + processBlocks(t, fh, testChain(511, 900)) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "blocks of the last merged-blocks file and above are kept") + assert.Equal(t, uint64(499), fh.mergedBlocks.lag(fh.forkable.LIBNum())) + + var fromLastMergedFile []*bstream.PreprocessedBlock + require.NoError(t, fh.forkable.CallWithBlocksFromNum(399, func(blocks []*bstream.PreprocessedBlock) { fromLastMergedFile = blocks }, false)) + require.NotEmpty(t, fromLastMergedFile) + assert.Equal(t, uint64(399), fromLastMergedFile[0].Block.Number) + + // the merger catches up + setMergedBlocksFiles(mergedBlocksStore, 400, 500, 600) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(901, 901)) + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "blocks below the last merged-blocks file are dropped") + assert.Equal(t, uint64(200), fh.mergedBlocks.lag(fh.forkable.LIBNum())) + + // back to keepFinalBlocks once the merged blocks are close to LIB + setMergedBlocksFiles(mergedBlocksStore, 700) + processBlocks(t, fh, testChain(902, 1010)) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(1011, 1011)) + assert.Equal(t, uint64(700), fh.forkable.LowestBlockNum()) + + setMergedBlocksFiles(mergedBlocksStore, 800, 900) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(1012, 1012)) + assert.Equal(t, uint64(810), fh.forkable.LowestBlockNum()) +} + +func TestForkableHub_MaxKeptFinalBlocks(t *testing.T) { + fh, _ := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{300}, WithMaxKeptFinalBlocks(350)) + require.NoError(t, fh.bootstrap()) + + processBlocks(t, fh, testChain(511, 900)) + assert.Equal(t, uint64(898-350), fh.forkable.LowestBlockNum()) +} + +func TestForkableHub_BootstrapFromLastMergedBlocksFile(t *testing.T) { + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}) + require.NoError(t, fh.bootstrap()) + + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "starts from the last merged-blocks file instead of 200 blocks below LIB") + assert.Equal(t, uint64(400), fh.mergedBlocks.nextBase.Load()) +} + +func TestForkableHub_BootstrapWithMergedBlocksUpToDate(t *testing.T) { + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300, 400, 500, 600, 700}) + + var lookups []string + mergedBlocksStore.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + lookups = append(lookups, base) + _, found := mergedBlocksStore.Files[base] + return found, nil + } + + require.NoError(t, fh.bootstrap()) + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum()) + assert.Equal(t, []string{"0000000600"}, lookups) + + // no lookup until LIB passes the end of the next file + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 799)) + assert.Equal(t, []string{"0000000600"}, lookups) + + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 800)) + assert.Equal(t, []string{"0000000600", "0000000700"}, lookups) + assert.Equal(t, uint64(800), fh.mergedBlocks.nextBase.Load()) +} + +func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { + store := dstore.NewMockStore(nil) + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 500} + + lowest, err := tracker.locate(context.Background(), 4800, 5010) + require.NoError(t, err) + assert.Equal(t, uint64(4600), lowest) + assert.Equal(t, uint64(4600), tracker.nextBase.Load()) +} + +func TestForkableHub_LiveBootstrapFromLastMergedBlocksFile(t *testing.T) { + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}) + + // no bootstrap from one-block files: the hub starts from its first live block + processBlocks(t, fh, testChain(901, 901)) + + require.True(t, fh.IsReady()) + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "starts from the last merged-blocks file instead of the live block's LIB") + assert.Equal(t, uint64(400), fh.mergedBlocks.nextBase.Load()) +} + +func TestMergedBlocksTracker_RetriesFailedLookups(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + defer func() { bootstrapRetryDelay = prev }() + + store := dstore.NewMockStore(nil) + setMergedBlocksFiles(store, 300) + + failures := 0 + store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + if failures > 0 { + failures-- + return false, errors.New("store unavailable") + } + _, found := store.Files[base] + return found, nil + } + + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} + + failures = bootstrapMaxAttempts - 1 + lowest, err := tracker.locate(context.Background(), 600, 800) + require.NoError(t, err) + assert.Equal(t, uint64(300), lowest) + + failures = bootstrapMaxAttempts + _, err = tracker.locate(context.Background(), 600, 800) + require.ErrorContains(t, err, "store unavailable") +} + +func TestForkableHub_MergedBlocksUpTo(t *testing.T) { + var nilHub *ForkableHub + _, ok := nilHub.MergedBlocksUpTo() + assert.False(t, ok) + + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}) + _, ok = fh.MergedBlocksUpTo() + assert.False(t, ok, "not known before the hub looked") + + require.NoError(t, fh.bootstrap()) + upTo, ok := fh.MergedBlocksUpTo() + assert.True(t, ok) + assert.Equal(t, uint64(400), upTo) +} From 308fdfa41bdf32b94ea11097d82db3aff6ca6a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 13:54:42 -0500 Subject: [PATCH 02/10] Keep hub blocks from the last merged file only A hub following the merged-blocks store no longer also applies a fixed depth below LIB, so there is one rule to reason about. The fixed depth remains for hubs without the store. --- CHANGELOG.md | 2 +- hub/hub.go | 50 +++++++++++++++++-------------- hub/merged_blocks.go | 42 ++++++++++---------------- hub/merged_blocks_test.go | 63 ++++++++++++++++++++++++++++----------- 4 files changed, 89 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 512254f..050036d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add `hub.WithMergedBlocksStore`: the hub keeps its final blocks until the merged-blocks file holding them exists, so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMaxKeptFinalBlocks`, `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. +- Add `hub.WithMergedBlocksStore(store, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. - Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. diff --git a/hub/hub.go b/hub/hub.go index 648c047..7b94f27 100644 --- a/hub/hub.go +++ b/hub/hub.go @@ -60,7 +60,6 @@ type ForkableHub struct { oneBlockDownloadConcurrency int mergedBlocks *mergedBlocksTracker // nil unless WithMergedBlocksStore is used - maxKeptFinalBlocks int mergedBlocksLagMetric forkable.Uint64Metric lastProgressLog time.Time @@ -110,11 +109,9 @@ func newForkableHub(liveSourceFactory bstream.SourceFactory, keepFinalBlocks int }, extraForkableOptions...) if hub.mergedBlocks != nil { - if hub.maxKeptFinalBlocks == 0 { - hub.maxKeptFinalBlocks = defaultMaxKeptFinalBlocksFactor * keepFinalBlocks - } - hub.mergedBlocks.bundleSize = bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) - hub.mergedBlocks.maxKeptFinalBlocks = uint64(max(hub.maxKeptFinalBlocks, keepFinalBlocks)) + t := hub.mergedBlocks + t.bundleSize = bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) + t.maxKeptFinalBlocks = max(t.maxKeptFinalBlocks, 2*t.bundleSize, uint64(keepFinalBlocks)) forkableOptions = append(forkableOptions, forkable.WithFinalBlocksPurgeLimit(hub.mergedBlocks.purgeLimit)) } @@ -420,15 +417,7 @@ func (h *ForkableHub) bootstrapOnce(ctx context.Context) error { if err != nil { return fmt.Errorf("parsing filename: %w", err) } - lowestBlockNum := substractAndRoundDownBlocks(refLibNum, uint64(h.keepFinalBlocks), bstream.DefaultMergedBlocksBundleSize) - if h.mergedBlocks != nil { - // Merged blocks that are behind move the starting point down to their last file. - if located, err := h.mergedBlocks.locate(ctx, lowestBlockNum, refLibNum); err != nil { - h.logger.Warn("cannot find the last merged-blocks file, bootstrapping as if merged blocks were up to date", zap.Error(err)) - } else { - lowestBlockNum = located - } - } + lowestBlockNum := h.lowestBlockNumToLoad(ctx, refLibNum) filenames, err := h.unknownOneBlockFiles(sortedOneBlocksFiles, lowestBlockNum) if err != nil { @@ -461,6 +450,27 @@ func (h *ForkableHub) bootstrapOnce(ctx context.Context) error { return nil } +// lowestBlockNumToLoad is the block from which a hub that holds nothing yet loads one-block +// files, given the LIB of the most recent block it knows. +// +// A hub following the merged-blocks store starts at its last file, so it covers every +// block the files do not hold. Otherwise it starts keepFinalBlocks below LIB, rounded down +// to a merged-blocks file boundary. +func (h *ForkableHub) lowestBlockNumToLoad(ctx context.Context, libNum uint64) uint64 { + bundleSize := bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) + if h.mergedBlocks == nil { + return substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), bundleSize) + } + + located, err := h.mergedBlocks.locate(ctx, substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), bundleSize), libNum) + if err != nil { + // two files below LIB is where the last merged-blocks file is when the merger keeps up + h.logger.Warn("cannot find the last merged-blocks file, assuming the merger keeps up", zap.Error(err)) + return substractAndRoundDownBlocks(libNum, 2*bundleSize, bundleSize) + } + return located +} + func (h *ForkableHub) Run() { liveSource := h.liveSourceFactory(h) liveSource.OnTerminating(h.reconnect) @@ -557,14 +567,8 @@ func (h *ForkableHub) linkLiveUsingOneBlocks(ctx context.Context, blk *pbbstream if !h.forkable.ForkDBHasLib() { lastKnownLib = blk.LibNum if h.mergedBlocks != nil { - // Same starting point as bootstrap: the last merged-blocks file, so the hub - // covers every block the files do not hold yet. - lowestBlockNum := substractAndRoundDownBlocks(blk.LibNum, uint64(h.keepFinalBlocks), bstream.DefaultMergedBlocksBundleSize) - if located, err := h.mergedBlocks.locate(ctx, lowestBlockNum, blk.LibNum); err != nil { - h.logger.Warn("cannot find the last merged-blocks file, linking from the live block's LIB", zap.Error(err)) - } else { - lastKnownLib = min(lastKnownLib, located) - } + // same starting point as bootstrap + lastKnownLib = min(lastKnownLib, h.lowestBlockNumToLoad(ctx, blk.LibNum)) } } diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go index dbf4230..7aacf12 100644 --- a/hub/merged_blocks.go +++ b/hub/merged_blocks.go @@ -13,10 +13,6 @@ import ( "go.uber.org/zap" ) -// defaultMaxKeptFinalBlocksFactor sets the default limit on the final blocks a hub keeps -// while merged blocks are behind, as a multiple of its keepFinalBlocks. -const defaultMaxKeptFinalBlocksFactor = 10 - // mergedBlocksPollInterval is how often the hub checks whether the next merged-blocks file // showed up. It is a var so tests can shrink it. var mergedBlocksPollInterval = 5 * time.Second @@ -26,28 +22,22 @@ var mergedBlocksPollInterval = 5 * time.Second const mergedBlocksBehindLogInterval = 30 * time.Second // WithMergedBlocksStore makes the hub keep its final blocks until they are available in a -// merged-blocks file of store, instead of dropping everything older than keepFinalBlocks -// below LIB. Without it, a merger falling more than keepFinalBlocks behind LIB leaves a -// range of blocks that neither the merged-blocks files nor the hub can serve, and every -// stream reading through the files waits there until the merger catches up. +// merged-blocks file of store: it holds every block from the start of the last +// merged-blocks file it found, so a stream reading the files can always continue on the +// hub. The hub is then built with a keepFinalBlocks of 0, which only matters to a hub +// without this option: a fixed depth below LIB leaves, when the merger falls further +// behind than that depth, a range of blocks that neither the files nor the hub can serve. // // The hub learns about new files with one FileExists call on the next expected file, at // most every 5 seconds and only once LIB passed the end of that file. // -// The extra blocks are held in memory, up to WithMaxKeptFinalBlocks. -func WithMergedBlocksStore(store dstore.Store) Option { +// maxKeptFinalBlocks limits the final blocks held in memory when the merger is behind or +// the store cannot be reached. Past it the oldest blocks are dropped even though no +// merged-blocks file holds them. It must cover at least two merged-blocks files, which is +// what the hub holds when the merger keeps up. +func WithMergedBlocksStore(store dstore.Store, maxKeptFinalBlocks int) Option { return func(h *ForkableHub) { - h.mergedBlocks = &mergedBlocksTracker{store: store} - } -} - -// WithMaxKeptFinalBlocks limits how many final blocks the hub keeps below LIB while it -// waits for merged blocks, see WithMergedBlocksStore. Past that limit the oldest blocks -// are dropped even though no merged-blocks file holds them. Defaults to 10 times the -// hub's keepFinalBlocks; a value below keepFinalBlocks is raised to it. -func WithMaxKeptFinalBlocks(count int) Option { - return func(h *ForkableHub) { - h.maxKeptFinalBlocks = count + h.mergedBlocks = &mergedBlocksTracker{store: store, maxKeptFinalBlocks: uint64(maxKeptFinalBlocks)} } } @@ -161,12 +151,9 @@ func (t *mergedBlocksTracker) lag(libNum uint64) uint64 { // last merged-blocks file and everything above: a stream reading the files joins the hub // on a block it received from a file, so the hub has to overlap the last one. func (t *mergedBlocksTracker) purgeLimit(libNum, cutoff uint64) uint64 { - if !t.known.Load() { - return cutoff - } - + // Nothing is dropped on account of merged blocks until the hub knows where they end. var limit uint64 - if nextBase := t.nextBase.Load(); nextBase > t.bundleSize { + if nextBase := t.nextBase.Load(); t.known.Load() && nextBase > t.bundleSize { limit = nextBase - t.bundleSize } if libNum > t.maxKeptFinalBlocks { @@ -225,7 +212,8 @@ func (h *ForkableHub) trackMergedBlocks() { h.mergedBlocksLagMetric.SetUint64(lag) } - if lag > uint64(h.keepFinalBlocks) && time.Since(lastBehindLog) >= mergedBlocksBehindLogInterval { + // a merger keeping up is at most one file behind LIB, plus the time it takes to write it + if lag > 3*t.bundleSize && time.Since(lastBehindLog) >= mergedBlocksBehindLogInterval { lastBehindLog = time.Now() lowest := h.forkable.LowestBlockNum() h.logger.Warn("merged blocks are behind, keeping more final blocks in memory to cover for them", diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go index 2d31988..1c922f5 100644 --- a/hub/merged_blocks_test.go +++ b/hub/merged_blocks_test.go @@ -29,7 +29,7 @@ func setMergedBlocksFiles(store *dstore.MockStore, bases ...uint64) { } } -func newMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBases []uint64, opts ...Option) (*ForkableHub, *dstore.MockStore) { +func newMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBases []uint64, maxKeptFinalBlocks int) (*ForkableHub, *dstore.MockStore) { t.Helper() prev := bstream.GetProtocolFirstStreamableBlock @@ -43,7 +43,7 @@ func newMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBa setMergedBlocksFiles(mergedBlocksStore, mergedBases...) lsf := bstream.NewTestSourceFactory() - fh := NewForkableHubWithOptions(lsf.NewSource, 200, oneBlocksStore, append([]Option{WithMergedBlocksStore(mergedBlocksStore)}, opts...)) + fh := NewForkableHubWithOptions(lsf.NewSource, 0, oneBlocksStore, []Option{WithMergedBlocksStore(mergedBlocksStore, maxKeptFinalBlocks)}) return fh, mergedBlocksStore } @@ -55,7 +55,7 @@ func processBlocks(t *testing.T, fh *ForkableHub, blocks []*pbbstream.Block) { } func TestForkableHub_KeepsFinalBlocksUntilMerged(t *testing.T) { - fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{100, 200, 300}) + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{100, 200, 300}, 2000) require.NoError(t, fh.bootstrap()) require.Equal(t, uint64(300), fh.forkable.LowestBlockNum()) @@ -77,7 +77,7 @@ func TestForkableHub_KeepsFinalBlocksUntilMerged(t *testing.T) { assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "blocks below the last merged-blocks file are dropped") assert.Equal(t, uint64(200), fh.mergedBlocks.lag(fh.forkable.LIBNum())) - // back to keepFinalBlocks once the merged blocks are close to LIB + // the hub follows the merged-blocks files as they show up setMergedBlocksFiles(mergedBlocksStore, 700) processBlocks(t, fh, testChain(902, 1010)) require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) @@ -87,11 +87,11 @@ func TestForkableHub_KeepsFinalBlocksUntilMerged(t *testing.T) { setMergedBlocksFiles(mergedBlocksStore, 800, 900) require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) processBlocks(t, fh, testChain(1012, 1012)) - assert.Equal(t, uint64(810), fh.forkable.LowestBlockNum()) + assert.Equal(t, uint64(900), fh.forkable.LowestBlockNum()) } func TestForkableHub_MaxKeptFinalBlocks(t *testing.T) { - fh, _ := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{300}, WithMaxKeptFinalBlocks(350)) + fh, _ := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{300}, 350) require.NoError(t, fh.bootstrap()) processBlocks(t, fh, testChain(511, 900)) @@ -99,15 +99,15 @@ func TestForkableHub_MaxKeptFinalBlocks(t *testing.T) { } func TestForkableHub_BootstrapFromLastMergedBlocksFile(t *testing.T) { - fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}) + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) require.NoError(t, fh.bootstrap()) - assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "starts from the last merged-blocks file instead of 200 blocks below LIB") + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "starts from the last merged-blocks file") assert.Equal(t, uint64(400), fh.mergedBlocks.nextBase.Load()) } func TestForkableHub_BootstrapWithMergedBlocksUpToDate(t *testing.T) { - fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300, 400, 500, 600, 700}) + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300, 400, 500, 600, 700}, 2000) var lookups []string mergedBlocksStore.FileExistsFunc = func(_ context.Context, base string) (bool, error) { @@ -117,15 +117,15 @@ func TestForkableHub_BootstrapWithMergedBlocksUpToDate(t *testing.T) { } require.NoError(t, fh.bootstrap()) - assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum()) - assert.Equal(t, []string{"0000000600"}, lookups) + assert.Equal(t, uint64(700), fh.forkable.LowestBlockNum()) + assert.Equal(t, []string{"0000000800", "0000000700"}, lookups) // no lookup until LIB passes the end of the next file - require.NoError(t, fh.mergedBlocks.advance(context.Background(), 799)) - assert.Equal(t, []string{"0000000600"}, lookups) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 899)) + assert.Equal(t, []string{"0000000800", "0000000700"}, lookups) - require.NoError(t, fh.mergedBlocks.advance(context.Background(), 800)) - assert.Equal(t, []string{"0000000600", "0000000700"}, lookups) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 900)) + assert.Equal(t, []string{"0000000800", "0000000700", "0000000800"}, lookups) assert.Equal(t, uint64(800), fh.mergedBlocks.nextBase.Load()) } @@ -140,7 +140,7 @@ func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { } func TestForkableHub_LiveBootstrapFromLastMergedBlocksFile(t *testing.T) { - fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}) + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) // no bootstrap from one-block files: the hub starts from its first live block processBlocks(t, fh, testChain(901, 901)) @@ -185,7 +185,7 @@ func TestForkableHub_MergedBlocksUpTo(t *testing.T) { _, ok := nilHub.MergedBlocksUpTo() assert.False(t, ok) - fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}) + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) _, ok = fh.MergedBlocksUpTo() assert.False(t, ok, "not known before the hub looked") @@ -194,3 +194,32 @@ func TestForkableHub_MergedBlocksUpTo(t *testing.T) { assert.True(t, ok) assert.Equal(t, uint64(400), upTo) } + +func TestMergedBlocksTracker_PurgeLimit(t *testing.T) { + tracker := &mergedBlocksTracker{bundleSize: 100, maxKeptFinalBlocks: 2000} + + assert.Equal(t, uint64(0), tracker.purgeLimit(1500, 1500), "nothing dropped before the merged blocks are located") + assert.Equal(t, uint64(3000), tracker.purgeLimit(5000, 5000), "except past maxKeptFinalBlocks") + + tracker.nextBase.Store(4800) + tracker.known.Store(true) + assert.Equal(t, uint64(4700), tracker.purgeLimit(5000, 5000), "keeps the last merged-blocks file") + assert.Equal(t, uint64(4600), tracker.purgeLimit(5000, 4600), "never above the forkable's own cutoff") +} + +func TestForkableHub_BootstrapWhenMergedBlocksStoreFails(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + defer func() { bootstrapRetryDelay = prev }() + + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) + mergedBlocksStore.FileExistsFunc = func(context.Context, string) (bool, error) { + return false, errors.New("store unavailable") + } + + require.NoError(t, fh.bootstrap()) + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "two merged-blocks files below LIB") + + processBlocks(t, fh, testChain(901, 1200)) + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "nothing is dropped while the merged blocks are not located") +} From c7b0faee9d01164967aba0214c208aec17b1834b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 13:55:58 -0500 Subject: [PATCH 03/10] Handle a hub starting before or far above merged blocks The lag counts from the first streamable block until the first merged-blocks file exists, and a hub booting further above the last file than it can hold searches for where merged blocks really end. --- hub/merged_blocks.go | 80 ++++++++++++++++++++++++++++++++++----- hub/merged_blocks_test.go | 72 +++++++++++++++++++++++++++++++++-- 2 files changed, 138 insertions(+), 14 deletions(-) diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go index 7aacf12..7f18259 100644 --- a/hub/merged_blocks.go +++ b/hub/merged_blocks.go @@ -87,13 +87,15 @@ func (t *mergedBlocksTracker) fileExists(ctx context.Context, base uint64) (exis // locate finds where the merged blocks end, looking down from the file that holds // lowestBlockNum, and returns the block the hub must start from for its blocks to overlap -// the last merged-blocks file. It does not go further than maxKeptFinalBlocks below libNum. +// the last merged-blocks file. That block is never further than maxKeptFinalBlocks below +// libNum, nor below the first streamable block. func (t *mergedBlocksTracker) locate(ctx context.Context, lowestBlockNum, libNum uint64) (uint64, error) { t.lookupLock.Lock() defer t.lookupLock.Unlock() - firstBase := bstream.GetProtocolFirstStreamableBlock / t.bundleSize * t.bundleSize - base := lowestBlockNum / t.bundleSize * t.bundleSize + firstBase := t.firstBase() + base := max(lowestBlockNum/t.bundleSize*t.bundleSize, firstBase) + nextBase := base for { exists, err := t.fileExists(ctx, base) @@ -101,20 +103,76 @@ func (t *mergedBlocksTracker) locate(ctx context.Context, lowestBlockNum, libNum return 0, err } if exists { - t.nextBase.Store(base + t.bundleSize) + nextBase = base + t.bundleSize break } - if base <= firstBase || libNum-min(libNum, base)+t.bundleSize > t.maxKeptFinalBlocks { - t.nextBase.Store(base) + if base <= firstBase { + // no merged-blocks file yet + nextBase = firstBase + break + } + if libNum-min(libNum, base)+t.bundleSize > t.maxKeptFinalBlocks { + // The merged blocks end lower than the hub can go. Where they end still + // matters to the lag metric and to MergedBlocksUpTo. + if nextBase, err = t.searchEnd(ctx, firstBase, base); err != nil { + return 0, err + } break } base -= t.bundleSize } + + t.nextBase.Store(nextBase) t.known.Store(true) return max(min(base, lowestBlockNum), bstream.GetProtocolFirstStreamableBlock), nil } +// firstBase is the base of the merged-blocks file holding the first streamable block. +func (t *mergedBlocksTracker) firstBase() uint64 { + return bstream.GetProtocolFirstStreamableBlock / t.bundleSize * t.bundleSize +} + +// searchEnd returns the base of the first missing merged-blocks file, knowing that the one +// at missing is not there and assuming the files are contiguous from firstBase. It probes +// further and further down until it finds a file, then narrows down between the two. +func (t *mergedBlocksTracker) searchEnd(ctx context.Context, firstBase, missing uint64) (uint64, error) { + found := firstBase + for step := t.bundleSize; ; step *= 2 { + probe := firstBase + if missing-firstBase > step { + probe = missing - step + } + + exists, err := t.fileExists(ctx, probe) + if err != nil { + return 0, err + } + if exists { + found = probe + break + } + if probe == firstBase { + return firstBase, nil + } + missing = probe + } + + for missing-found > t.bundleSize { + middle := found + (missing-found)/t.bundleSize/2*t.bundleSize + exists, err := t.fileExists(ctx, middle) + if err != nil { + return 0, err + } + if exists { + found = middle + } else { + missing = middle + } + } + return missing, nil +} + // advance moves nextBase over the files that showed up. A file is only looked up once LIB // passed its last block, since the merger cannot have written it before. func (t *mergedBlocksTracker) advance(ctx context.Context, libNum uint64) error { @@ -138,13 +196,15 @@ func (t *mergedBlocksTracker) advance(ctx context.Context, libNum uint64) error } } -// lag is how many blocks libNum is ahead of the last merged block. +// lag is how many blocks libNum is ahead of the last merged block: the final blocks that +// are not in a merged-blocks file. func (t *mergedBlocksTracker) lag(libNum uint64) uint64 { - nextBase := t.nextBase.Load() - if libNum < nextBase { + // before the first merged-blocks file, the lag counts from the first streamable block + firstUnmerged := max(t.nextBase.Load(), bstream.GetProtocolFirstStreamableBlock) + if libNum < firstUnmerged { return 0 } - return libNum + 1 - nextBase + return libNum + 1 - firstUnmerged } // purgeLimit is a forkable.WithFinalBlocksPurgeLimit function. It keeps the blocks of the diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go index 1c922f5..0399133 100644 --- a/hub/merged_blocks_test.go +++ b/hub/merged_blocks_test.go @@ -130,13 +130,77 @@ func TestForkableHub_BootstrapWithMergedBlocksUpToDate(t *testing.T) { } func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { + bases := func(from, to uint64) (out []uint64) { + for base := from; base <= to; base += 100 { + out = append(out, base) + } + return out + } + + tests := []struct { + name string + firstStreamableBlock uint64 + mergedBases []uint64 + expectedLowest uint64 + expectedNextBase uint64 + }{ + {"no merged-blocks file at all", 0, nil, 4600, 0}, + {"no merged-blocks file at all, chain starting above 0", 1250, nil, 4600, 1200}, + {"only the first file", 0, bases(0, 0), 4600, 100}, + {"merged blocks end far below", 0, bases(0, 1200), 4600, 1300}, + {"merged blocks end far below, chain starting above 0", 1250, bases(1200, 3300), 4600, 3400}, + {"merged blocks end one file below reach", 0, bases(0, 4500), 4600, 4600}, + {"merged blocks end within reach", 0, bases(0, 4600), 4600, 4700}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = test.firstStreamableBlock + defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + + store := dstore.NewMockStore(nil) + setMergedBlocksFiles(store, test.mergedBases...) + lookups := 0 + store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + lookups++ + _, found := store.Files[base] + return found, nil + } + + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 500} + + lowest, err := tracker.locate(context.Background(), 4800, 5010) + require.NoError(t, err) + assert.Equal(t, test.expectedLowest, lowest) + assert.Equal(t, test.expectedNextBase, tracker.nextBase.Load()) + assert.LessOrEqual(t, lookups, 20) + }) + } +} + +func TestMergedBlocksTracker_BeforeFirstMergedBlocksFile(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 150 + defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + store := dstore.NewMockStore(nil) - tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 500} + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} - lowest, err := tracker.locate(context.Background(), 4800, 5010) + lowest, err := tracker.locate(context.Background(), 150, 180) require.NoError(t, err) - assert.Equal(t, uint64(4600), lowest) - assert.Equal(t, uint64(4600), tracker.nextBase.Load()) + assert.Equal(t, uint64(150), lowest, "never below the first streamable block") + assert.Equal(t, uint64(100), tracker.nextBase.Load()) + assert.Equal(t, uint64(31), tracker.lag(180), "counted from the first streamable block") + assert.Equal(t, uint64(0), tracker.purgeLimit(450, 450), "everything is kept until the first file exists") + + // the first file holds blocks 150 to 199 + setMergedBlocksFiles(store, 100) + require.NoError(t, tracker.advance(context.Background(), 199)) + assert.Equal(t, uint64(100), tracker.nextBase.Load(), "not looked up before LIB passed its end") + require.NoError(t, tracker.advance(context.Background(), 205)) + assert.Equal(t, uint64(200), tracker.nextBase.Load()) + assert.Equal(t, uint64(6), tracker.lag(205)) } func TestForkableHub_LiveBootstrapFromLastMergedBlocksFile(t *testing.T) { From 4e89aa95299648942e27d607b737aabb6c9559d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 15:05:32 -0500 Subject: [PATCH 04/10] Take the merged-blocks file size in WithMergedBlocksStore The hub used the process-wide default, so a hub reading files of another size looked for files that never exist and stopped following the merger. --- CHANGELOG.md | 2 +- hub/hub.go | 5 +++-- hub/merged_blocks.go | 11 +++++++++-- hub/merged_blocks_test.go | 31 ++++++++++++++++++++++++++++++- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 050036d..3eb0ae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add `hub.WithMergedBlocksStore(store, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. +- Add `hub.WithMergedBlocksStore(store, bundleSize, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. - Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. diff --git a/hub/hub.go b/hub/hub.go index 7b94f27..bcd637c 100644 --- a/hub/hub.go +++ b/hub/hub.go @@ -110,7 +110,6 @@ func newForkableHub(liveSourceFactory bstream.SourceFactory, keepFinalBlocks int if hub.mergedBlocks != nil { t := hub.mergedBlocks - t.bundleSize = bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) t.maxKeptFinalBlocks = max(t.maxKeptFinalBlocks, 2*t.bundleSize, uint64(keepFinalBlocks)) forkableOptions = append(forkableOptions, forkable.WithFinalBlocksPurgeLimit(hub.mergedBlocks.purgeLimit)) } @@ -457,11 +456,13 @@ func (h *ForkableHub) bootstrapOnce(ctx context.Context) error { // block the files do not hold. Otherwise it starts keepFinalBlocks below LIB, rounded down // to a merged-blocks file boundary. func (h *ForkableHub) lowestBlockNumToLoad(ctx context.Context, libNum uint64) uint64 { - bundleSize := bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) if h.mergedBlocks == nil { + bundleSize := bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) return substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), bundleSize) } + bundleSize := h.mergedBlocks.bundleSize + located, err := h.mergedBlocks.locate(ctx, substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), bundleSize), libNum) if err != nil { // two files below LIB is where the last merged-blocks file is when the merger keeps up diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go index 7f18259..9976914 100644 --- a/hub/merged_blocks.go +++ b/hub/merged_blocks.go @@ -31,13 +31,20 @@ const mergedBlocksBehindLogInterval = 30 * time.Second // The hub learns about new files with one FileExists call on the next expected file, at // most every 5 seconds and only once LIB passed the end of that file. // +// bundleSize is the number of blocks per merged-blocks file in store (0 means +// bstream.DefaultMergedBlocksBundleSize). +// // maxKeptFinalBlocks limits the final blocks held in memory when the merger is behind or // the store cannot be reached. Past it the oldest blocks are dropped even though no // merged-blocks file holds them. It must cover at least two merged-blocks files, which is // what the hub holds when the merger keeps up. -func WithMergedBlocksStore(store dstore.Store, maxKeptFinalBlocks int) Option { +func WithMergedBlocksStore(store dstore.Store, bundleSize uint64, maxKeptFinalBlocks int) Option { return func(h *ForkableHub) { - h.mergedBlocks = &mergedBlocksTracker{store: store, maxKeptFinalBlocks: uint64(maxKeptFinalBlocks)} + h.mergedBlocks = &mergedBlocksTracker{ + store: store, + bundleSize: bstream.SanitizeBundleSize(bundleSize), + maxKeptFinalBlocks: uint64(maxKeptFinalBlocks), + } } } diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go index 0399133..4ba0b20 100644 --- a/hub/merged_blocks_test.go +++ b/hub/merged_blocks_test.go @@ -43,7 +43,7 @@ func newMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBa setMergedBlocksFiles(mergedBlocksStore, mergedBases...) lsf := bstream.NewTestSourceFactory() - fh := NewForkableHubWithOptions(lsf.NewSource, 0, oneBlocksStore, []Option{WithMergedBlocksStore(mergedBlocksStore, maxKeptFinalBlocks)}) + fh := NewForkableHubWithOptions(lsf.NewSource, 0, oneBlocksStore, []Option{WithMergedBlocksStore(mergedBlocksStore, 100, maxKeptFinalBlocks)}) return fh, mergedBlocksStore } @@ -90,6 +90,35 @@ func TestForkableHub_KeepsFinalBlocksUntilMerged(t *testing.T) { assert.Equal(t, uint64(900), fh.forkable.LowestBlockNum()) } +func TestForkableHub_MergedBlocksBundleSize(t *testing.T) { + require.Equal(t, uint64(100), bstream.DefaultMergedBlocksBundleSize) + + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + oneBlocksStore := dstore.NewMockStore(nil) + AddToMockStore(t, oneBlocksStore, testChain(2000, 3100)...) + mergedBlocksStore := dstore.NewMockStore(nil) + setMergedBlocksFiles(mergedBlocksStore, 0, 1000, 2000) + + fh := NewForkableHubWithOptions(bstream.NewTestSourceFactory().NewSource, 0, oneBlocksStore, []Option{WithMergedBlocksStore(mergedBlocksStore, 1000, 20000)}) + require.NoError(t, fh.bootstrap()) + assert.Equal(t, uint64(2000), fh.forkable.LowestBlockNum()) + upTo, ok := fh.MergedBlocksUpTo() + require.True(t, ok) + assert.Equal(t, uint64(3000), upTo) + + setMergedBlocksFiles(mergedBlocksStore, 3000) + processBlocks(t, fh, testChain(3101, 4010)) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(4011, 4011)) + + upTo, _ = fh.MergedBlocksUpTo() + assert.Equal(t, uint64(4000), upTo) + assert.Equal(t, uint64(3000), fh.forkable.LowestBlockNum()) +} + func TestForkableHub_MaxKeptFinalBlocks(t *testing.T) { fh, _ := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{300}, 350) require.NoError(t, fh.bootstrap()) From e9defd91bd8cac43f71e685465b563f683bf898c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 15:25:43 -0500 Subject: [PATCH 05/10] Search merged blocks once while linking live blocks Each live block the hub could not link searched the merged-blocks store again until the hub had a LIB. It now reuses the last result, and waits 5s after a failed search. --- hub/hub.go | 21 ++++++++--- hub/merged_blocks.go | 30 ++++++++++++++++ hub/merged_blocks_test.go | 76 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/hub/hub.go b/hub/hub.go index bcd637c..d418e59 100644 --- a/hub/hub.go +++ b/hub/hub.go @@ -455,19 +455,32 @@ func (h *ForkableHub) bootstrapOnce(ctx context.Context) error { // A hub following the merged-blocks store starts at its last file, so it covers every // block the files do not hold. Otherwise it starts keepFinalBlocks below LIB, rounded down // to a merged-blocks file boundary. +// +// It runs for every live block the hub cannot link until it has a LIB, so the store is +// only searched until a search succeeds, and at most once per mergedBlocksPollInterval +// while searches fail. func (h *ForkableHub) lowestBlockNumToLoad(ctx context.Context, libNum uint64) uint64 { if h.mergedBlocks == nil { bundleSize := bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) return substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), bundleSize) } - bundleSize := h.mergedBlocks.bundleSize + t := h.mergedBlocks + if start, ok := t.knownStart(libNum); ok { + return start + } + + // two files below LIB is where the last merged-blocks file is when the merger keeps up + assumedStart := substractAndRoundDownBlocks(libNum, 2*t.bundleSize, t.bundleSize) + if t.lookupFailedRecently() { + return assumedStart + } - located, err := h.mergedBlocks.locate(ctx, substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), bundleSize), libNum) + located, err := t.locate(ctx, substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), t.bundleSize), libNum) if err != nil { - // two files below LIB is where the last merged-blocks file is when the merger keeps up + t.lookupFailedAt.Store(time.Now().UnixNano()) h.logger.Warn("cannot find the last merged-blocks file, assuming the merger keeps up", zap.Error(err)) - return substractAndRoundDownBlocks(libNum, 2*bundleSize, bundleSize) + return assumedStart } return located } diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go index 9976914..e1efd7c 100644 --- a/hub/merged_blocks.go +++ b/hub/merged_blocks.go @@ -71,6 +71,10 @@ type mergedBlocksTracker struct { // store. Every file below it was seen, or sits under where the hub started looking. nextBase atomic.Uint64 known atomic.Bool + + // lookupFailedAt is when the hub last failed to locate the merged blocks while + // starting, in Unix nanoseconds. + lookupFailedAt atomic.Int64 } func mergedBlocksFilename(base uint64) string { @@ -135,6 +139,32 @@ func (t *mergedBlocksTracker) locate(ctx context.Context, lowestBlockNum, libNum return max(min(base, lowestBlockNum), bstream.GetProtocolFirstStreamableBlock), nil } +// knownStart is the block the hub must start from at a LIB of libNum, computed from what +// the last successful locate found instead of searching the store: the start of the last +// merged-blocks file, never further than maxKeptFinalBlocks below libNum, nor below the +// first streamable block. ok is false when no locate succeeded yet. +func (t *mergedBlocksTracker) knownStart(libNum uint64) (start uint64, ok bool) { + if !t.known.Load() { + return 0, false + } + + start = t.firstBase() + if nextBase := t.nextBase.Load(); nextBase >= start+t.bundleSize { + start = nextBase - t.bundleSize + } + if libNum > t.maxKeptFinalBlocks { + start = max(start, libNum-t.maxKeptFinalBlocks) + } + return max(start, bstream.GetProtocolFirstStreamableBlock), true +} + +// lookupFailedRecently reports whether locating the merged blocks failed less than +// mergedBlocksPollInterval ago. +func (t *mergedBlocksTracker) lookupFailedRecently() bool { + failedAt := t.lookupFailedAt.Load() + return failedAt != 0 && time.Since(time.Unix(0, failedAt)) < mergedBlocksPollInterval +} + // firstBase is the base of the merged-blocks file holding the first streamable block. func (t *mergedBlocksTracker) firstBase() uint64 { return bstream.GetProtocolFirstStreamableBlock / t.bundleSize * t.bundleSize diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go index 4ba0b20..f5d1f0b 100644 --- a/hub/merged_blocks_test.go +++ b/hub/merged_blocks_test.go @@ -316,3 +316,79 @@ func TestForkableHub_BootstrapWhenMergedBlocksStoreFails(t *testing.T) { processBlocks(t, fh, testChain(901, 1200)) assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "nothing is dropped while the merged blocks are not located") } + +// liveChainWithoutLIB returns blocks from..to whose LIB is 100 blocks below them, so a hub +// receiving only these blocks has no LIB in its forkDB. +func liveChainWithoutLIB(from, to uint64) (out []*pbbstream.Block) { + id := func(num uint64) string { return fmt.Sprintf("%08x", num) } + for num := from; num <= to; num++ { + out = append(out, bstream.TestBlockWithLIBNum(id(num), id(num-1), num-100)) + } + return out +} + +func countFileExists(store *dstore.MockStore, fail bool) *int { + calls := 0 + store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + calls++ + if fail { + return false, errors.New("store unavailable") + } + _, found := store.Files[base] + return found, nil + } + return &calls +} + +func TestForkableHub_LiveLinkingSearchesMergedBlocksOnce(t *testing.T) { + fh, mergedBlocksStore := newMergedBlocksTestHub(t, nil, []uint64{100, 200, 300}, 2000) + calls := countFileExists(mergedBlocksStore, false) + + processBlocks(t, fh, liveChainWithoutLIB(901, 901)) + require.False(t, fh.forkable.ForkDBHasLib()) + searchCalls := *calls + require.NotZero(t, searchCalls) + + processBlocks(t, fh, liveChainWithoutLIB(902, 950)) + require.False(t, fh.forkable.ForkDBHasLib()) + assert.Equal(t, searchCalls, *calls, "later blocks reuse what the first search found") + assert.Equal(t, uint64(300), func() uint64 { start, _ := fh.mergedBlocks.knownStart(850); return start }()) +} + +func TestForkableHub_LiveLinkingWaitsAfterFailedSearch(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + defer func() { bootstrapRetryDelay = prev }() + + fh, mergedBlocksStore := newMergedBlocksTestHub(t, nil, []uint64{100, 200, 300}, 2000) + calls := countFileExists(mergedBlocksStore, true) + + processBlocks(t, fh, liveChainWithoutLIB(901, 950)) + assert.Equal(t, bootstrapMaxAttempts, *calls, "one failed search, then none until mergedBlocksPollInterval passed") + + fh.mergedBlocks.lookupFailedAt.Store(time.Now().Add(-mergedBlocksPollInterval).UnixNano()) + processBlocks(t, fh, liveChainWithoutLIB(951, 951)) + assert.Equal(t, 2*bootstrapMaxAttempts, *calls, "searches again once mergedBlocksPollInterval passed") +} + +func TestMergedBlocksTracker_KnownStart(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + + tracker := &mergedBlocksTracker{bundleSize: 100, maxKeptFinalBlocks: 2000} + _, ok := tracker.knownStart(800) + assert.False(t, ok, "not known before a search") + + tracker.known.Store(true) + tracker.nextBase.Store(0) + start, _ := tracker.knownStart(800) + assert.Equal(t, uint64(1), start, "no merged-blocks file: first streamable block") + + tracker.nextBase.Store(400) + start, _ = tracker.knownStart(800) + assert.Equal(t, uint64(300), start, "start of the last merged-blocks file") + + start, _ = tracker.knownStart(5000) + assert.Equal(t, uint64(3000), start, "never more than maxKeptFinalBlocks below LIB") +} From c68a8e15d10adecbbf9cd217798d7e43505020b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 15:39:39 -0500 Subject: [PATCH 06/10] Stop looking for merged blocks past the hub cap Below maxKeptFinalBlocks the search could miss files that do not start at the first streamable block, leaving the hub waiting on a file the merger never writes. --- hub/merged_blocks.go | 64 +++++++-------------------------- hub/merged_blocks_test.go | 76 +++++++++++++++++++++++++++++++++++---- 2 files changed, 83 insertions(+), 57 deletions(-) diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go index e1efd7c..ac957a4 100644 --- a/hub/merged_blocks.go +++ b/hub/merged_blocks.go @@ -51,7 +51,9 @@ func WithMergedBlocksStore(store dstore.Store, bundleSize uint64, maxKeptFinalBl // WithMergedBlocksLagMetric reports how many blocks LIB is ahead of the last block found // in the merged-blocks store. It needs WithMergedBlocksStore. With a merger keeping up, // the value moves between 1 and a little over one merged-blocks file; it keeps growing -// when the merger is stopped or too slow. +// when the merger is stopped or too slow. The hub does not look for merged-blocks files +// more than maxKeptFinalBlocks below LIB, so a hub starting while the merger is further +// behind reports about maxKeptFinalBlocks at first. func WithMergedBlocksLagMetric(metric forkable.Uint64Metric) Option { return func(h *ForkableHub) { h.mergedBlocksLagMetric = metric @@ -68,7 +70,8 @@ type mergedBlocksTracker struct { lookupLock sync.Mutex // serializes the store lookups // nextBase is the base block number of the first merged-blocks file not seen in the - // store. Every file below it was seen, or sits under where the hub started looking. + // store. Every file below it was seen, or sits under where the hub started looking: + // the hub never looks more than maxKeptFinalBlocks below LIB. nextBase atomic.Uint64 known atomic.Bool @@ -99,7 +102,7 @@ func (t *mergedBlocksTracker) fileExists(ctx context.Context, base uint64) (exis // locate finds where the merged blocks end, looking down from the file that holds // lowestBlockNum, and returns the block the hub must start from for its blocks to overlap // the last merged-blocks file. That block is never further than maxKeptFinalBlocks below -// libNum, nor below the first streamable block. +// libNum, nor below the first streamable block, and neither is where locate stops looking. func (t *mergedBlocksTracker) locate(ctx context.Context, lowestBlockNum, libNum uint64) (uint64, error) { t.lookupLock.Lock() defer t.lookupLock.Unlock() @@ -123,11 +126,9 @@ func (t *mergedBlocksTracker) locate(ctx context.Context, lowestBlockNum, libNum break } if libNum-min(libNum, base)+t.bundleSize > t.maxKeptFinalBlocks { - // The merged blocks end lower than the hub can go. Where they end still - // matters to the lag metric and to MergedBlocksUpTo. - if nextBase, err = t.searchEnd(ctx, firstBase, base); err != nil { - return 0, err - } + // The merged blocks end lower than the hub can go. The merger writes files in + // order, so it writes the one at base once it gets there. + nextBase = base break } base -= t.bundleSize @@ -170,46 +171,6 @@ func (t *mergedBlocksTracker) firstBase() uint64 { return bstream.GetProtocolFirstStreamableBlock / t.bundleSize * t.bundleSize } -// searchEnd returns the base of the first missing merged-blocks file, knowing that the one -// at missing is not there and assuming the files are contiguous from firstBase. It probes -// further and further down until it finds a file, then narrows down between the two. -func (t *mergedBlocksTracker) searchEnd(ctx context.Context, firstBase, missing uint64) (uint64, error) { - found := firstBase - for step := t.bundleSize; ; step *= 2 { - probe := firstBase - if missing-firstBase > step { - probe = missing - step - } - - exists, err := t.fileExists(ctx, probe) - if err != nil { - return 0, err - } - if exists { - found = probe - break - } - if probe == firstBase { - return firstBase, nil - } - missing = probe - } - - for missing-found > t.bundleSize { - middle := found + (missing-found)/t.bundleSize/2*t.bundleSize - exists, err := t.fileExists(ctx, middle) - if err != nil { - return 0, err - } - if exists { - found = middle - } else { - missing = middle - } - } - return missing, nil -} - // advance moves nextBase over the files that showed up. A file is only looked up once LIB // passed its last block, since the merger cannot have written it before. func (t *mergedBlocksTracker) advance(ctx context.Context, libNum uint64) error { @@ -260,8 +221,9 @@ func (t *mergedBlocksTracker) purgeLimit(libNum, cutoff uint64) uint64 { } // MergedBlocksUpTo returns the first block number the hub has not found in a merged-blocks -// file: every block below it can be read from the merged-blocks store. ok is false when -// the hub does not follow that store (see WithMergedBlocksStore) or has not looked yet. +// file, at most maxKeptFinalBlocks below LIB when the hub started: the files end there or +// lower. ok is false when the hub does not follow that store (see WithMergedBlocksStore) +// or has not looked yet. func (h *ForkableHub) MergedBlocksUpTo() (blockNum uint64, ok bool) { if h == nil || h.mergedBlocks == nil || !h.mergedBlocks.known.Load() { return 0, false @@ -294,7 +256,7 @@ func (h *ForkableHub) trackMergedBlocks() { libNum := h.forkable.LIBNum() if !t.known.Load() { - if _, err := t.locate(ctx, h.forkable.LowestBlockNum(), libNum); err != nil { + if _, err := t.locate(ctx, libNum, libNum); err != nil { h.logger.Warn("cannot find the last merged-blocks file", zap.Error(err)) continue } diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go index f5d1f0b..261d99f 100644 --- a/hub/merged_blocks_test.go +++ b/hub/merged_blocks_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync" "testing" "time" @@ -173,11 +174,11 @@ func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { expectedLowest uint64 expectedNextBase uint64 }{ - {"no merged-blocks file at all", 0, nil, 4600, 0}, - {"no merged-blocks file at all, chain starting above 0", 1250, nil, 4600, 1200}, - {"only the first file", 0, bases(0, 0), 4600, 100}, - {"merged blocks end far below", 0, bases(0, 1200), 4600, 1300}, - {"merged blocks end far below, chain starting above 0", 1250, bases(1200, 3300), 4600, 3400}, + {"no merged-blocks file at all", 0, nil, 4600, 4600}, + {"no merged-blocks file at all, chain starting above 0", 1250, nil, 4600, 4600}, + {"only the first file", 0, bases(0, 0), 4600, 4600}, + {"merged blocks end far below", 0, bases(0, 1200), 4600, 4600}, + {"merged blocks end far below, chain starting above 0", 1250, bases(1200, 3300), 4600, 4600}, {"merged blocks end one file below reach", 0, bases(0, 4500), 4600, 4600}, {"merged blocks end within reach", 0, bases(0, 4600), 4600, 4700}, } @@ -203,7 +204,7 @@ func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { require.NoError(t, err) assert.Equal(t, test.expectedLowest, lowest) assert.Equal(t, test.expectedNextBase, tracker.nextBase.Load()) - assert.LessOrEqual(t, lookups, 20) + assert.LessOrEqual(t, lookups, 3, "stops looking at maxKeptFinalBlocks below LIB") }) } } @@ -392,3 +393,66 @@ func TestMergedBlocksTracker_KnownStart(t *testing.T) { start, _ = tracker.knownStart(5000) assert.Equal(t, uint64(3000), start, "never more than maxKeptFinalBlocks below LIB") } + +func TestMergedBlocksTracker_FollowsMergerFromFarBehind(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 0 + defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + + // the store does not start at the first streamable block, and the merger is far behind + store := dstore.NewMockStore(nil) + for base := uint64(10000); base <= 10400; base += 100 { + setMergedBlocksFiles(store, base) + } + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} + + lowest, err := tracker.locate(context.Background(), 20010, 20010) + require.NoError(t, err) + assert.Equal(t, uint64(18100), lowest) + assert.Equal(t, uint64(18100), tracker.nextBase.Load(), "looks no further than maxKeptFinalBlocks below LIB") + + // the merger catches up, writing its files in order + for base := uint64(10500); base <= 19900; base += 100 { + setMergedBlocksFiles(store, base) + } + require.NoError(t, tracker.advance(context.Background(), 20010)) + assert.Equal(t, uint64(20000), tracker.nextBase.Load()) + assert.Equal(t, uint64(11), tracker.lag(20010)) +} + +func TestForkableHub_TrackerRetryLooksDownFromLIB(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + defer func() { bootstrapRetryDelay = prev }() + prevPoll := mergedBlocksPollInterval + mergedBlocksPollInterval = 10 * time.Millisecond + defer func() { mergedBlocksPollInterval = prevPoll }() + + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) + failing := true + var lookedLock sync.Mutex + var looked []string + mergedBlocksStore.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + if failing { + return false, errors.New("store unavailable") + } + lookedLock.Lock() + defer lookedLock.Unlock() + looked = append(looked, base) + _, found := mergedBlocksStore.Files[base] + return found, nil + } + + require.NoError(t, fh.bootstrap()) + close(fh.Ready) + failing = false + go fh.trackMergedBlocks() + defer fh.Shutdown(nil) + + require.Eventually(t, func() bool { _, ok := fh.MergedBlocksUpTo(); return ok }, time.Second, 5*time.Millisecond) + upTo, _ := fh.MergedBlocksUpTo() + assert.Equal(t, uint64(400), upTo) + lookedLock.Lock() + defer lookedLock.Unlock() + assert.Equal(t, mergedBlocksFilename(800), looked[0], "starts from LIB's file") +} From 231e6735c295ddbe9afb825f8b5909851af14f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 15:46:23 -0500 Subject: [PATCH 07/10] Log missing one-block files, time out lookups A hub waiting on missing one-block files now says which blocks it waits for. Merged-blocks lookups stop after 30s and on shutdown instead of hanging the hub. --- CHANGELOG.md | 1 + hub/hub.go | 54 +++++++++++++ hub/merged_blocks.go | 46 ++++++++--- hub/merged_blocks_test.go | 157 +++++++++++++++++++++++++++++++++++--- 4 files changed, 238 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eb0ae9..d9810ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add `hub.WithMergedBlocksStore(store, bundleSize, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. +- A hub that is not ready because one-block files are missing between the blocks it loaded and the live blocks logs the missing range every 30s. It waits for those files instead of starting above them, so streams going from merged blocks to live blocks never skip blocks. - Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. diff --git a/hub/hub.go b/hub/hub.go index d418e59..e521c17 100644 --- a/hub/hub.go +++ b/hub/hub.go @@ -63,6 +63,11 @@ type ForkableHub struct { mergedBlocksLagMetric forkable.Uint64Metric lastProgressLog time.Time + + // while not ready, since when live blocks do not link to the blocks loaded from + // one-block files, and when that was last logged + notLinkingSince time.Time + lastNotLinkingLog time.Time } const defaultOneBlockDownloadConcurrency = 32 @@ -470,6 +475,9 @@ func (h *ForkableHub) lowestBlockNumToLoad(ctx context.Context, libNum uint64) u return start } + ctx, cancel := context.WithTimeout(ctx, mergedBlocksLookupTimeout) + defer cancel() + // two files below LIB is where the last merged-blocks file is when the merger keeps up assumedStart := substractAndRoundDownBlocks(libNum, 2*t.bundleSize, t.bundleSize) if t.lookupFailedRecently() { @@ -549,6 +557,9 @@ func (h *ForkableHub) ProcessBlock(blk *pbbstream.Block, obj any) error { // limit would be reached after a quarter of the blocks it names — and the check is // there to catch a gap the one-block store can no longer bridge, which the block's // final partial reports just as well. A real block and a final flash block count. + if !h.IsReady() { + h.logNotLinking(blk) + } if h.maxConsecutiveUnlinkableBlocks != 0 && !isNonFinalPartial(blk) { h.consecutiveUnlinkableBlocks++ h.logger.Warn("block not linkable after one-block lookup", @@ -562,6 +573,7 @@ func (h *ForkableHub) ProcessBlock(blk *pbbstream.Block, obj any) error { } } else { // linkable h.consecutiveUnlinkableBlocks = 0 + h.notLinkingSince = time.Time{} if !h.IsReady() { h.logger.Info("Hub is ready") close(h.Ready) @@ -575,6 +587,48 @@ func (h *ForkableHub) ProcessBlock(blk *pbbstream.Block, obj any) error { var errRestartRequired = errors.New("restart required") +// notLinkingLogInterval is how long live blocks must fail to link to the blocks a hub +// loaded from one-block files before it says so, and how often it says it again. +var notLinkingLogInterval = 30 * time.Second + +// logNotLinking explains why a hub that loaded blocks from one-block files is not ready: +// the live blocks do not link to them, because one-block files are missing in between. +// The hub does not skip over the missing blocks, since streams going from merged blocks +// to live blocks would miss them; it gets ready once the files show up. +func (h *ForkableHub) logNotLinking(blk *pbbstream.Block) { + if !h.forkable.ForkDBHasLib() { + // still bootstrapping from live blocks + return + } + + now := time.Now() + if h.notLinkingSince.IsZero() { + h.notLinkingSince = now + } + if now.Sub(h.notLinkingSince) < notLinkingLogInterval || now.Sub(h.lastNotLinkingLog) < notLinkingLogInterval { + return + } + h.lastNotLinkingLog = now + + // the lowest block the live blocks reach by following parent links + lowestLive := blk + for { + parent := h.forkable.GetBlockByHash(lowestLive.ParentId) + if parent == nil { + break + } + lowestLive = parent + } + + h.logger.Warn("hub not ready: one-block files are missing between the blocks loaded from one-block files and the live blocks, waiting for them to show up", + zap.Uint64("missing_from_block", h.forkable.HeadNum()+1), + zap.Uint64("missing_up_to_block", lowestLive.Number-1), + zap.Uint64("lib_num", h.forkable.LIBNum()), + zap.Uint64("live_block_num", blk.Number), + zap.Duration("not_linking_for", now.Sub(h.notLinkingSince)), + ) +} + func (h *ForkableHub) linkLiveUsingOneBlocks(ctx context.Context, blk *pbbstream.Block) error { lastKnownLib := h.forkable.LowestBlockNum() diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go index ac957a4..5f4680e 100644 --- a/hub/merged_blocks.go +++ b/hub/merged_blocks.go @@ -17,6 +17,11 @@ import ( // showed up. It is a var so tests can shrink it. var mergedBlocksPollInterval = 5 * time.Second +// mergedBlocksLookupTimeout bounds each search of the merged-blocks store, so a store call +// that hangs does not stop the hub from following the merger. It is a var so tests can +// shrink it. +var mergedBlocksLookupTimeout = 30 * time.Second + // mergedBlocksBehindLogInterval is how often the hub warns that it keeps extra final // blocks because merged blocks are behind. const mergedBlocksBehindLogInterval = 30 * time.Second @@ -90,7 +95,11 @@ func mergedBlocksFilename(base uint64) string { func (t *mergedBlocksTracker) fileExists(ctx context.Context, base uint64) (exists bool, err error) { for attempt := range bootstrapMaxAttempts { if attempt > 0 { - time.Sleep(bootstrapRetryDelay) + select { + case <-ctx.Done(): + return false, fmt.Errorf("looking up merged-blocks file %s: %w", mergedBlocksFilename(base), ctx.Err()) + case <-time.After(bootstrapRetryDelay): + } } if exists, err = t.store.FileExists(ctx, mergedBlocksFilename(base)); err == nil { return exists, nil @@ -231,11 +240,35 @@ func (h *ForkableHub) MergedBlocksUpTo() (blockNum uint64, ok bool) { return h.mergedBlocks.nextBase.Load(), true } +// followMergedBlocks locates the merged blocks if no search succeeded yet, then moves +// over the files that showed up. It returns false when the merged blocks are not located. +func (h *ForkableHub) followMergedBlocks(ctx context.Context, libNum uint64) bool { + ctx, cancel := context.WithTimeout(ctx, mergedBlocksLookupTimeout) + defer cancel() + + t := h.mergedBlocks + if !t.known.Load() { + if _, err := t.locate(ctx, libNum, libNum); err != nil { + h.logger.Warn("cannot find the last merged-blocks file", zap.Error(err)) + return false + } + } + + if err := t.advance(ctx, libNum); err != nil { + h.logger.Warn("cannot check for new merged-blocks files", zap.Error(err)) + } + return true +} + // trackMergedBlocks keeps the merged blocks tracker and the lag metric up to date until // the hub terminates. func (h *ForkableHub) trackMergedBlocks() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + go func() { + <-h.Terminating() + cancel() + }() ticker := time.NewTicker(mergedBlocksPollInterval) defer ticker.Stop() @@ -255,15 +288,8 @@ func (h *ForkableHub) trackMergedBlocks() { t := h.mergedBlocks libNum := h.forkable.LIBNum() - if !t.known.Load() { - if _, err := t.locate(ctx, libNum, libNum); err != nil { - h.logger.Warn("cannot find the last merged-blocks file", zap.Error(err)) - continue - } - } - - if err := t.advance(ctx, libNum); err != nil { - h.logger.Warn("cannot check for new merged-blocks files", zap.Error(err)) + if !h.followMergedBlocks(ctx, libNum) { + continue } lag := t.lag(libNum) diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go index 261d99f..865dca1 100644 --- a/hub/merged_blocks_test.go +++ b/hub/merged_blocks_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "testing" "time" @@ -13,6 +14,9 @@ import ( "github.com/streamingfast/dstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" ) // testChain returns blocks from..to, each one final two blocks later. @@ -212,7 +216,7 @@ func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { func TestMergedBlocksTracker_BeforeFirstMergedBlocksFile(t *testing.T) { prev := bstream.GetProtocolFirstStreamableBlock bstream.GetProtocolFirstStreamableBlock = 150 - defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) store := dstore.NewMockStore(nil) tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} @@ -247,7 +251,7 @@ func TestForkableHub_LiveBootstrapFromLastMergedBlocksFile(t *testing.T) { func TestMergedBlocksTracker_RetriesFailedLookups(t *testing.T) { prev := bootstrapRetryDelay bootstrapRetryDelay = time.Millisecond - defer func() { bootstrapRetryDelay = prev }() + t.Cleanup(func() { bootstrapRetryDelay = prev }) store := dstore.NewMockStore(nil) setMergedBlocksFiles(store, 300) @@ -304,7 +308,7 @@ func TestMergedBlocksTracker_PurgeLimit(t *testing.T) { func TestForkableHub_BootstrapWhenMergedBlocksStoreFails(t *testing.T) { prev := bootstrapRetryDelay bootstrapRetryDelay = time.Millisecond - defer func() { bootstrapRetryDelay = prev }() + t.Cleanup(func() { bootstrapRetryDelay = prev }) fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) mergedBlocksStore.FileExistsFunc = func(context.Context, string) (bool, error) { @@ -359,7 +363,7 @@ func TestForkableHub_LiveLinkingSearchesMergedBlocksOnce(t *testing.T) { func TestForkableHub_LiveLinkingWaitsAfterFailedSearch(t *testing.T) { prev := bootstrapRetryDelay bootstrapRetryDelay = time.Millisecond - defer func() { bootstrapRetryDelay = prev }() + t.Cleanup(func() { bootstrapRetryDelay = prev }) fh, mergedBlocksStore := newMergedBlocksTestHub(t, nil, []uint64{100, 200, 300}, 2000) calls := countFileExists(mergedBlocksStore, true) @@ -375,7 +379,7 @@ func TestForkableHub_LiveLinkingWaitsAfterFailedSearch(t *testing.T) { func TestMergedBlocksTracker_KnownStart(t *testing.T) { prev := bstream.GetProtocolFirstStreamableBlock bstream.GetProtocolFirstStreamableBlock = 1 - defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) tracker := &mergedBlocksTracker{bundleSize: 100, maxKeptFinalBlocks: 2000} _, ok := tracker.knownStart(800) @@ -397,7 +401,7 @@ func TestMergedBlocksTracker_KnownStart(t *testing.T) { func TestMergedBlocksTracker_FollowsMergerFromFarBehind(t *testing.T) { prev := bstream.GetProtocolFirstStreamableBlock bstream.GetProtocolFirstStreamableBlock = 0 - defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) // the store does not start at the first streamable block, and the merger is far behind store := dstore.NewMockStore(nil) @@ -423,10 +427,10 @@ func TestMergedBlocksTracker_FollowsMergerFromFarBehind(t *testing.T) { func TestForkableHub_TrackerRetryLooksDownFromLIB(t *testing.T) { prev := bootstrapRetryDelay bootstrapRetryDelay = time.Millisecond - defer func() { bootstrapRetryDelay = prev }() + t.Cleanup(func() { bootstrapRetryDelay = prev }) prevPoll := mergedBlocksPollInterval mergedBlocksPollInterval = 10 * time.Millisecond - defer func() { mergedBlocksPollInterval = prevPoll }() + t.Cleanup(func() { mergedBlocksPollInterval = prevPoll }) fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) failing := true @@ -446,8 +450,7 @@ func TestForkableHub_TrackerRetryLooksDownFromLIB(t *testing.T) { require.NoError(t, fh.bootstrap()) close(fh.Ready) failing = false - go fh.trackMergedBlocks() - defer fh.Shutdown(nil) + runTracker(t, fh) require.Eventually(t, func() bool { _, ok := fh.MergedBlocksUpTo(); return ok }, time.Second, 5*time.Millisecond) upTo, _ := fh.MergedBlocksUpTo() @@ -456,3 +459,137 @@ func TestForkableHub_TrackerRetryLooksDownFromLIB(t *testing.T) { defer lookedLock.Unlock() assert.Equal(t, mergedBlocksFilename(800), looked[0], "starts from LIB's file") } + +// runTracker runs the hub's merged-blocks tracker until the test ends, and waits for it +// to stop before the test restores the globals it reads. +func runTracker(t *testing.T, fh *ForkableHub) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + fh.trackMergedBlocks() + }() + t.Cleanup(func() { + fh.Shutdown(nil) + <-done + }) +} + +func newObservedMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBases []uint64, extra ...Option) (*ForkableHub, *dstore.MockStore, *dstore.MockStore, *observer.ObservedLogs) { + t.Helper() + + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + oneBlocksStore := dstore.NewMockStore(nil) + AddToMockStore(t, oneBlocksStore, oneBlocks...) + mergedBlocksStore := dstore.NewMockStore(nil) + setMergedBlocksFiles(mergedBlocksStore, mergedBases...) + + core, logs := observer.New(zapcore.WarnLevel) + options := append([]Option{WithMergedBlocksStore(mergedBlocksStore, 100, 2000), WithLogger(zap.New(core))}, extra...) + fh := NewForkableHubWithOptions(bstream.NewTestSourceFactory().NewSource, 0, oneBlocksStore, options) + return fh, oneBlocksStore, mergedBlocksStore, logs +} + +func TestForkableHub_LogsMissingOneBlocksWhileNotReady(t *testing.T) { + prev := notLinkingLogInterval + notLinkingLogInterval = 0 + t.Cleanup(func() { notLinkingLogInterval = prev }) + + // one-block files 400 to 449 are missing, so the merger stopped after file 300 + fh, oneBlocksStore, _, logs := newObservedMergedBlocksTestHub(t, append(testChain(300, 399), testChain(450, 900)...), []uint64{100, 200, 300}) + + require.Error(t, fh.bootstrap()) + processBlocks(t, fh, testChain(901, 905)) + require.False(t, fh.IsReady(), "does not skip over the missing blocks") + + notLinking := logs.FilterMessageSnippet("one-block files are missing").All() + require.NotEmpty(t, notLinking) + fields := notLinking[0].ContextMap() + assert.Equal(t, uint64(400), fields["missing_from_block"]) + assert.Equal(t, uint64(449), fields["missing_up_to_block"]) + + AddToMockStore(t, oneBlocksStore, testChain(400, 449)...) + processBlocks(t, fh, testChain(906, 906)) + assert.True(t, fh.IsReady(), "gets ready once the missing files show up") + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum()) +} + +func TestForkableHub_NoMissingOneBlocksLogWhenLinking(t *testing.T) { + prev := notLinkingLogInterval + notLinkingLogInterval = 0 + t.Cleanup(func() { notLinkingLogInterval = prev }) + + fh, _, _, logs := newObservedMergedBlocksTestHub(t, testChain(300, 900), []uint64{100, 200, 300}) + require.NoError(t, fh.bootstrap()) + close(fh.Ready) + processBlocks(t, fh, testChain(901, 905)) + assert.Empty(t, logs.FilterMessageSnippet("one-block files are missing").All()) +} + +type testUint64Metric struct{ value atomic.Uint64 } + +func (m *testUint64Metric) SetUint64(value uint64) { m.value.Store(value) } + +func TestForkableHub_TrackerSetsLagAndWarnsWhenBehind(t *testing.T) { + prev := mergedBlocksPollInterval + mergedBlocksPollInterval = 5 * time.Millisecond + t.Cleanup(func() { mergedBlocksPollInterval = prev }) + + metric := &testUint64Metric{} + fh, _, _, logs := newObservedMergedBlocksTestHub(t, testChain(300, 900), []uint64{100, 200, 300}, WithMergedBlocksLagMetric(metric)) + require.NoError(t, fh.bootstrap()) + close(fh.Ready) + runTracker(t, fh) + + // LIB is 898 and the merged blocks end at 399 + require.Eventually(t, func() bool { return metric.value.Load() == 499 }, time.Second, 5*time.Millisecond) + require.Eventually(t, func() bool { return logs.FilterMessageSnippet("merged blocks are behind").Len() > 0 }, time.Second, 5*time.Millisecond) +} + +func TestForkableHub_MergedBlocksLookupTimesOut(t *testing.T) { + prevTimeout := mergedBlocksLookupTimeout + mergedBlocksLookupTimeout = 20 * time.Millisecond + t.Cleanup(func() { mergedBlocksLookupTimeout = prevTimeout }) + prevPoll := mergedBlocksPollInterval + mergedBlocksPollInterval = 5 * time.Millisecond + t.Cleanup(func() { mergedBlocksPollInterval = prevPoll }) + + fh, _, mergedBlocksStore, logs := newObservedMergedBlocksTestHub(t, testChain(300, 900), []uint64{100, 200, 300}) + mergedBlocksStore.FileExistsFunc = func(ctx context.Context, _ string) (bool, error) { + <-ctx.Done() // a store call that hangs until its context ends + return false, ctx.Err() + } + + start := time.Now() + require.NoError(t, fh.bootstrap()) + assert.Less(t, time.Since(start), time.Second, "the boot search gives up after mergedBlocksLookupTimeout") + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "two merged-blocks files below LIB") + + close(fh.Ready) + runTracker(t, fh) + require.Eventually(t, func() bool { + return logs.FilterMessageSnippet("cannot find the last merged-blocks file").Len() >= 2 + }, time.Second, 5*time.Millisecond, "the tracker keeps retrying") +} + +func TestMergedBlocksTracker_FileExistsStopsWaitingOnCancel(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Hour + t.Cleanup(func() { bootstrapRetryDelay = prev }) + + store := dstore.NewMockStore(nil) + store.FileExistsFunc = func(context.Context, string) (bool, error) { + return false, errors.New("store unavailable") + } + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + _, err := tracker.fileExists(ctx, 100) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(start), time.Second) +} From 612545aea38e9300100c19a13c063a5ead50a8ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 16:13:19 -0500 Subject: [PATCH 08/10] List merged-blocks files instead of probing each one A hub starting while the merger is behind looked up one file per call going down from LIB, and one per call going up when the merger caught up. Both are now one listing. The listing also shows a file written ahead of a missing one, which is no longer taken for the end of merged blocks. --- hub/hub.go | 2 +- hub/merged_blocks.go | 132 +++++++++++++++------- hub/merged_blocks_test.go | 232 ++++++++++++++++++++++++++------------ 3 files changed, 255 insertions(+), 111 deletions(-) diff --git a/hub/hub.go b/hub/hub.go index e521c17..5273279 100644 --- a/hub/hub.go +++ b/hub/hub.go @@ -484,7 +484,7 @@ func (h *ForkableHub) lowestBlockNumToLoad(ctx context.Context, libNum uint64) u return assumedStart } - located, err := t.locate(ctx, substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), t.bundleSize), libNum) + located, err := t.locate(ctx, libNum) if err != nil { t.lookupFailedAt.Store(time.Now().UnixNano()) h.logger.Warn("cannot find the last merged-blocks file, assuming the merger keeps up", zap.Error(err)) diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go index 5f4680e..2645836 100644 --- a/hub/merged_blocks.go +++ b/hub/merged_blocks.go @@ -3,6 +3,8 @@ package hub import ( "context" "fmt" + "strconv" + "strings" "sync" "sync/atomic" "time" @@ -33,8 +35,10 @@ const mergedBlocksBehindLogInterval = 30 * time.Second // without this option: a fixed depth below LIB leaves, when the merger falls further // behind than that depth, a range of blocks that neither the files nor the hub can serve. // -// The hub learns about new files with one FileExists call on the next expected file, at -// most every 5 seconds and only once LIB passed the end of that file. +// The hub finds where merged blocks end with one listing of the store when it starts. It +// then learns about new files with one FileExists call on the next expected file, at most +// every 5 seconds and only once LIB passed the end of that file, or with one listing when +// LIB is far enough for several new files to be there, as when the merger catches up. // // bundleSize is the number of blocks per merged-blocks file in store (0 means // bstream.DefaultMergedBlocksBundleSize). @@ -108,45 +112,80 @@ func (t *mergedBlocksTracker) fileExists(ctx context.Context, base uint64) (exis return false, fmt.Errorf("looking up merged-blocks file %s: %w", mergedBlocksFilename(base), err) } -// locate finds where the merged blocks end, looking down from the file that holds -// lowestBlockNum, and returns the block the hub must start from for its blocks to overlap -// the last merged-blocks file. That block is never further than maxKeptFinalBlocks below -// libNum, nor below the first streamable block, and neither is where locate stops looking. -func (t *mergedBlocksTracker) locate(ctx context.Context, lowestBlockNum, libNum uint64) (uint64, error) { +// firstRunFrom lists the store from the merged-blocks file starting at base, and returns +// the first run of consecutive files it holds there: the base of its first file and the +// base that follows its last one. ok is false when the store holds no file from base on. +// +// It is one listing whatever the number of files, stopped at the end of the run. A listing +// that fails is retried like a file lookup is. +func (t *mergedBlocksTracker) firstRunFrom(ctx context.Context, base uint64) (first, next uint64, ok bool, err error) { + for attempt := range bootstrapMaxAttempts { + if attempt > 0 { + select { + case <-ctx.Done(): + return 0, 0, false, fmt.Errorf("listing merged-blocks files from %s: %w", mergedBlocksFilename(base), ctx.Err()) + case <-time.After(bootstrapRetryDelay): + } + } + + ok = false + err = t.store.WalkFrom(ctx, "", mergedBlocksFilename(base), func(filename string) error { + name, _, _ := strings.Cut(filename, ".") + fileBase, parseErr := strconv.ParseUint(name, 10, 64) + if parseErr != nil || fileBase%t.bundleSize != 0 || fileBase < base { + return nil // not a merged-blocks file of this store's size + } + if !ok { + first, next, ok = fileBase, fileBase+t.bundleSize, true + return nil + } + if fileBase != next { + return dstore.StopIteration + } + next += t.bundleSize + return nil + }) + if err == nil { + return first, next, ok, nil + } + } + return 0, 0, false, fmt.Errorf("listing merged-blocks files from %s: %w", mergedBlocksFilename(base), err) +} + +// locate finds where the merged blocks end and returns the block the hub must start from +// at a LIB of libNum for its blocks to overlap the last merged-blocks file. That block is +// never further than maxKeptFinalBlocks below libNum, nor below the first streamable +// block, and neither is where locate looks. +// +// It takes the first run of consecutive files in that range, so a file the merger wrote +// ahead of a lower one that is still missing is not taken for the end of merged blocks. +func (t *mergedBlocksTracker) locate(ctx context.Context, libNum uint64) (uint64, error) { t.lookupLock.Lock() defer t.lookupLock.Unlock() - firstBase := t.firstBase() - base := max(lowestBlockNum/t.bundleSize*t.bundleSize, firstBase) - nextBase := base + // the lowest file the hub can start from + floor := t.firstBase() + if reach := libNum + t.bundleSize; reach > t.maxKeptFinalBlocks { + floor = max(floor, (reach-t.maxKeptFinalBlocks-1)/t.bundleSize*t.bundleSize) + } + + _, next, ok, err := t.firstRunFrom(ctx, floor) + if err != nil { + return 0, err + } - for { - exists, err := t.fileExists(ctx, base) - if err != nil { - return 0, err - } - if exists { - nextBase = base + t.bundleSize - break - } - if base <= firstBase { - // no merged-blocks file yet - nextBase = firstBase - break - } - if libNum-min(libNum, base)+t.bundleSize > t.maxKeptFinalBlocks { - // The merged blocks end lower than the hub can go. The merger writes files in - // order, so it writes the one at base once it gets there. - nextBase = base - break - } - base -= t.bundleSize + // Without a file in reach, either there is none yet, or the merged blocks end lower + // than the hub can go: the merger writes files in order, so it writes the one at floor + // once it gets there. + start, nextBase := floor, floor + if ok { + start, nextBase = next-t.bundleSize, next } t.nextBase.Store(nextBase) t.known.Store(true) - return max(min(base, lowestBlockNum), bstream.GetProtocolFirstStreamableBlock), nil + return max(start, bstream.GetProtocolFirstStreamableBlock), nil } // knownStart is the block the hub must start from at a LIB of libNum, computed from what @@ -186,21 +225,32 @@ func (t *mergedBlocksTracker) advance(ctx context.Context, libNum uint64) error t.lookupLock.Lock() defer t.lookupLock.Unlock() - for { - base := t.nextBase.Load() - if libNum < base+t.bundleSize { - return nil - } + base := t.nextBase.Load() + if libNum < base+t.bundleSize { + return nil + } + // only the next file can be there: one lookup + if libNum < base+2*t.bundleSize { exists, err := t.fileExists(ctx, base) if err != nil { return err } - if !exists { - return nil + if exists { + t.nextBase.Store(base + t.bundleSize) } - t.nextBase.Store(base + t.bundleSize) + return nil + } + + // several files can be there, as when the merger catches up: one listing + first, next, ok, err := t.firstRunFrom(ctx, base) + if err != nil { + return err + } + if ok && first == base { + t.nextBase.Store(next) } + return nil } // lag is how many blocks libNum is ahead of the last merged block: the final blocks that @@ -248,7 +298,7 @@ func (h *ForkableHub) followMergedBlocks(ctx context.Context, libNum uint64) boo t := h.mergedBlocks if !t.known.Load() { - if _, err := t.locate(ctx, libNum, libNum); err != nil { + if _, err := t.locate(ctx, libNum); err != nil { h.logger.Warn("cannot find the last merged-blocks file", zap.Error(err)) return false } diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go index 865dca1..0d2b01a 100644 --- a/hub/merged_blocks_test.go +++ b/hub/merged_blocks_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sort" "sync" "sync/atomic" "testing" @@ -19,6 +20,75 @@ import ( "go.uber.org/zap/zaptest/observer" ) +// storeObserver records the calls the hub makes on a merged-blocks store, as +// "exists " for a lookup and "list " for a listing from that file, and makes +// them fail while failing is set. +type storeObserver struct { + sync.Mutex + calls []string + failing bool +} + +func observeStore(store *dstore.MockStore) *storeObserver { + o := &storeObserver{} + record := func(call string) error { + o.Lock() + defer o.Unlock() + o.calls = append(o.calls, call) + if o.failing { + return errors.New("store unavailable") + } + return nil + } + + store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + if err := record("exists " + base); err != nil { + return false, err + } + _, found := store.Files[base] + return found, nil + } + store.WalkFromFunc = func(ctx context.Context, prefix, startingPoint string, f func(filename string) error) error { + if err := record("list " + startingPoint); err != nil { + return err + } + return listMockStoreFrom(store, startingPoint, f) + } + return o +} + +// listMockStoreFrom is what MockStore.WalkFrom does without a WalkFromFunc. +func listMockStoreFrom(store *dstore.MockStore, startingPoint string, f func(filename string) error) error { + var names []string + for name := range store.Files { + if name >= startingPoint { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + if err := f(name); err != nil { + if errors.Is(err, dstore.StopIteration) { + return nil + } + return err + } + } + return nil +} + +func (o *storeObserver) setFailing(failing bool) { + o.Lock() + defer o.Unlock() + o.failing = failing +} + +func (o *storeObserver) seen() []string { + o.Lock() + defer o.Unlock() + return append([]string(nil), o.calls...) +} + // testChain returns blocks from..to, each one final two blocks later. func testChain(from, to uint64) (out []*pbbstream.Block) { id := func(num uint64) string { return fmt.Sprintf("%08x", num) } @@ -142,25 +212,26 @@ func TestForkableHub_BootstrapFromLastMergedBlocksFile(t *testing.T) { func TestForkableHub_BootstrapWithMergedBlocksUpToDate(t *testing.T) { fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300, 400, 500, 600, 700}, 2000) - - var lookups []string - mergedBlocksStore.FileExistsFunc = func(_ context.Context, base string) (bool, error) { - lookups = append(lookups, base) - _, found := mergedBlocksStore.Files[base] - return found, nil - } + store := observeStore(mergedBlocksStore) require.NoError(t, fh.bootstrap()) assert.Equal(t, uint64(700), fh.forkable.LowestBlockNum()) - assert.Equal(t, []string{"0000000800", "0000000700"}, lookups) + assert.Equal(t, []string{"list 0000000000"}, store.seen(), "one listing whatever the number of files") - // no lookup until LIB passes the end of the next file + // nothing until LIB passes the end of the next file require.NoError(t, fh.mergedBlocks.advance(context.Background(), 899)) - assert.Equal(t, []string{"0000000800", "0000000700"}, lookups) + assert.Equal(t, []string{"list 0000000000"}, store.seen()) + // one lookup while only the next file can be there require.NoError(t, fh.mergedBlocks.advance(context.Background(), 900)) - assert.Equal(t, []string{"0000000800", "0000000700", "0000000800"}, lookups) + assert.Equal(t, []string{"list 0000000000", "exists 0000000800"}, store.seen()) assert.Equal(t, uint64(800), fh.mergedBlocks.nextBase.Load()) + + // one listing once several files can be there + setMergedBlocksFiles(mergedBlocksStore, 800, 900, 1000) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 1150)) + assert.Equal(t, []string{"list 0000000000", "exists 0000000800", "list 0000000800"}, store.seen()) + assert.Equal(t, uint64(1100), fh.mergedBlocks.nextBase.Load()) } func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { @@ -195,20 +266,15 @@ func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { store := dstore.NewMockStore(nil) setMergedBlocksFiles(store, test.mergedBases...) - lookups := 0 - store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { - lookups++ - _, found := store.Files[base] - return found, nil - } + observed := observeStore(store) tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 500} - lowest, err := tracker.locate(context.Background(), 4800, 5010) + lowest, err := tracker.locate(context.Background(), 5010) require.NoError(t, err) assert.Equal(t, test.expectedLowest, lowest) assert.Equal(t, test.expectedNextBase, tracker.nextBase.Load()) - assert.LessOrEqual(t, lookups, 3, "stops looking at maxKeptFinalBlocks below LIB") + assert.Len(t, observed.seen(), 1, "one listing, from maxKeptFinalBlocks below LIB") }) } } @@ -221,7 +287,7 @@ func TestMergedBlocksTracker_BeforeFirstMergedBlocksFile(t *testing.T) { store := dstore.NewMockStore(nil) tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} - lowest, err := tracker.locate(context.Background(), 150, 180) + lowest, err := tracker.locate(context.Background(), 180) require.NoError(t, err) assert.Equal(t, uint64(150), lowest, "never below the first streamable block") assert.Equal(t, uint64(100), tracker.nextBase.Load()) @@ -257,24 +323,23 @@ func TestMergedBlocksTracker_RetriesFailedLookups(t *testing.T) { setMergedBlocksFiles(store, 300) failures := 0 - store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + store.WalkFromFunc = func(_ context.Context, _, startingPoint string, f func(filename string) error) error { if failures > 0 { failures-- - return false, errors.New("store unavailable") + return errors.New("store unavailable") } - _, found := store.Files[base] - return found, nil + return listMockStoreFrom(store, startingPoint, f) } tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} failures = bootstrapMaxAttempts - 1 - lowest, err := tracker.locate(context.Background(), 600, 800) + lowest, err := tracker.locate(context.Background(), 800) require.NoError(t, err) assert.Equal(t, uint64(300), lowest) failures = bootstrapMaxAttempts - _, err = tracker.locate(context.Background(), 600, 800) + _, err = tracker.locate(context.Background(), 800) require.ErrorContains(t, err, "store unavailable") } @@ -311,9 +376,7 @@ func TestForkableHub_BootstrapWhenMergedBlocksStoreFails(t *testing.T) { t.Cleanup(func() { bootstrapRetryDelay = prev }) fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) - mergedBlocksStore.FileExistsFunc = func(context.Context, string) (bool, error) { - return false, errors.New("store unavailable") - } + observeStore(mergedBlocksStore).setFailing(true) require.NoError(t, fh.bootstrap()) assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "two merged-blocks files below LIB") @@ -332,31 +395,18 @@ func liveChainWithoutLIB(from, to uint64) (out []*pbbstream.Block) { return out } -func countFileExists(store *dstore.MockStore, fail bool) *int { - calls := 0 - store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { - calls++ - if fail { - return false, errors.New("store unavailable") - } - _, found := store.Files[base] - return found, nil - } - return &calls -} - func TestForkableHub_LiveLinkingSearchesMergedBlocksOnce(t *testing.T) { fh, mergedBlocksStore := newMergedBlocksTestHub(t, nil, []uint64{100, 200, 300}, 2000) - calls := countFileExists(mergedBlocksStore, false) + store := observeStore(mergedBlocksStore) processBlocks(t, fh, liveChainWithoutLIB(901, 901)) require.False(t, fh.forkable.ForkDBHasLib()) - searchCalls := *calls + searchCalls := len(store.seen()) require.NotZero(t, searchCalls) processBlocks(t, fh, liveChainWithoutLIB(902, 950)) require.False(t, fh.forkable.ForkDBHasLib()) - assert.Equal(t, searchCalls, *calls, "later blocks reuse what the first search found") + assert.Len(t, store.seen(), searchCalls, "later blocks reuse what the first search found") assert.Equal(t, uint64(300), func() uint64 { start, _ := fh.mergedBlocks.knownStart(850); return start }()) } @@ -366,14 +416,15 @@ func TestForkableHub_LiveLinkingWaitsAfterFailedSearch(t *testing.T) { t.Cleanup(func() { bootstrapRetryDelay = prev }) fh, mergedBlocksStore := newMergedBlocksTestHub(t, nil, []uint64{100, 200, 300}, 2000) - calls := countFileExists(mergedBlocksStore, true) + store := observeStore(mergedBlocksStore) + store.setFailing(true) processBlocks(t, fh, liveChainWithoutLIB(901, 950)) - assert.Equal(t, bootstrapMaxAttempts, *calls, "one failed search, then none until mergedBlocksPollInterval passed") + assert.Len(t, store.seen(), bootstrapMaxAttempts, "one failed search, then none until mergedBlocksPollInterval passed") fh.mergedBlocks.lookupFailedAt.Store(time.Now().Add(-mergedBlocksPollInterval).UnixNano()) processBlocks(t, fh, liveChainWithoutLIB(951, 951)) - assert.Equal(t, 2*bootstrapMaxAttempts, *calls, "searches again once mergedBlocksPollInterval passed") + assert.Len(t, store.seen(), 2*bootstrapMaxAttempts, "searches again once mergedBlocksPollInterval passed") } func TestMergedBlocksTracker_KnownStart(t *testing.T) { @@ -410,7 +461,7 @@ func TestMergedBlocksTracker_FollowsMergerFromFarBehind(t *testing.T) { } tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} - lowest, err := tracker.locate(context.Background(), 20010, 20010) + lowest, err := tracker.locate(context.Background(), 20010) require.NoError(t, err) assert.Equal(t, uint64(18100), lowest) assert.Equal(t, uint64(18100), tracker.nextBase.Load(), "looks no further than maxKeptFinalBlocks below LIB") @@ -424,7 +475,7 @@ func TestMergedBlocksTracker_FollowsMergerFromFarBehind(t *testing.T) { assert.Equal(t, uint64(11), tracker.lag(20010)) } -func TestForkableHub_TrackerRetryLooksDownFromLIB(t *testing.T) { +func TestForkableHub_TrackerLocatesOnceStoreAnswers(t *testing.T) { prev := bootstrapRetryDelay bootstrapRetryDelay = time.Millisecond t.Cleanup(func() { bootstrapRetryDelay = prev }) @@ -433,31 +484,20 @@ func TestForkableHub_TrackerRetryLooksDownFromLIB(t *testing.T) { t.Cleanup(func() { mergedBlocksPollInterval = prevPoll }) fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) - failing := true - var lookedLock sync.Mutex - var looked []string - mergedBlocksStore.FileExistsFunc = func(_ context.Context, base string) (bool, error) { - if failing { - return false, errors.New("store unavailable") - } - lookedLock.Lock() - defer lookedLock.Unlock() - looked = append(looked, base) - _, found := mergedBlocksStore.Files[base] - return found, nil - } + store := observeStore(mergedBlocksStore) + store.setFailing(true) require.NoError(t, fh.bootstrap()) + _, ok := fh.MergedBlocksUpTo() + require.False(t, ok) + close(fh.Ready) - failing = false + store.setFailing(false) runTracker(t, fh) require.Eventually(t, func() bool { _, ok := fh.MergedBlocksUpTo(); return ok }, time.Second, 5*time.Millisecond) upTo, _ := fh.MergedBlocksUpTo() assert.Equal(t, uint64(400), upTo) - lookedLock.Lock() - defer lookedLock.Unlock() - assert.Equal(t, mergedBlocksFilename(800), looked[0], "starts from LIB's file") } // runTracker runs the hub's merged-blocks tracker until the test ends, and waits for it @@ -558,9 +598,9 @@ func TestForkableHub_MergedBlocksLookupTimesOut(t *testing.T) { t.Cleanup(func() { mergedBlocksPollInterval = prevPoll }) fh, _, mergedBlocksStore, logs := newObservedMergedBlocksTestHub(t, testChain(300, 900), []uint64{100, 200, 300}) - mergedBlocksStore.FileExistsFunc = func(ctx context.Context, _ string) (bool, error) { + mergedBlocksStore.WalkFromFunc = func(ctx context.Context, _, _ string, _ func(string) error) error { <-ctx.Done() // a store call that hangs until its context ends - return false, ctx.Err() + return ctx.Err() } start := time.Now() @@ -593,3 +633,57 @@ func TestMergedBlocksTracker_FileExistsStopsWaitingOnCancel(t *testing.T) { require.ErrorIs(t, err, context.DeadlineExceeded) assert.Less(t, time.Since(start), time.Second) } + +func TestMergedBlocksTracker_ListsConsecutiveFiles(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 0 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + newTracker := func(bases ...uint64) (*mergedBlocksTracker, *dstore.MockStore) { + store := dstore.NewMockStore(nil) + setMergedBlocksFiles(store, bases...) + return &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000}, store + } + + t.Run("a file written ahead of a missing one is not the end of merged blocks", func(t *testing.T) { + tracker, _ := newTracker(300, 400, 500, 700) + start, err := tracker.locate(context.Background(), 950) + require.NoError(t, err) + assert.Equal(t, uint64(500), start) + assert.Equal(t, uint64(600), tracker.nextBase.Load()) + }) + + t.Run("a store that does not start at the first streamable block", func(t *testing.T) { + tracker, _ := newTracker(600, 700) + start, err := tracker.locate(context.Background(), 950) + require.NoError(t, err) + assert.Equal(t, uint64(700), start) + assert.Equal(t, uint64(800), tracker.nextBase.Load()) + }) + + t.Run("other files of the store are ignored", func(t *testing.T) { + tracker, store := newTracker(300, 400) + store.SetFile("0000000450", []byte("not on a file boundary")) + store.SetFile("0000000500.tmp", []byte("merged")) + store.SetFile("index", []byte("not a number")) + start, err := tracker.locate(context.Background(), 950) + require.NoError(t, err) + assert.Equal(t, uint64(500), start) + assert.Equal(t, uint64(600), tracker.nextBase.Load()) + }) + + t.Run("following a merger that catches up stops at the first missing file", func(t *testing.T) { + tracker, store := newTracker(300) + _, err := tracker.locate(context.Background(), 2000) + require.NoError(t, err) + require.Equal(t, uint64(400), tracker.nextBase.Load()) + + setMergedBlocksFiles(store, 500, 600) + require.NoError(t, tracker.advance(context.Background(), 2000)) + assert.Equal(t, uint64(400), tracker.nextBase.Load(), "the next file is not there") + + setMergedBlocksFiles(store, 400, 800) + require.NoError(t, tracker.advance(context.Background(), 2000)) + assert.Equal(t, uint64(700), tracker.nextBase.Load()) + }) +} From 2dad87c33b2a018730ca1b4e077b7dc02e5924df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 16:18:43 -0500 Subject: [PATCH 09/10] Fix two hub tests that fail under the race detector One appended to a slice from concurrent downloads. The other started its consumer before pushing blocks, so which partial blocks got skipped depended on timing. --- hub/hub_test.go | 7 +++++++ hub/subscription_test.go | 13 +++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hub/hub_test.go b/hub/hub_test.go index 09bd58f..923f1d6 100644 --- a/hub/hub_test.go +++ b/hub/hub_test.go @@ -8,6 +8,7 @@ import ( "io" "strconv" "strings" + "sync" "testing" "time" @@ -369,8 +370,12 @@ func TestForkableHub_LinkLiveUsingOneBlocks_StopsAtLiveBlock(t *testing.T) { } AddToMockStore(t, testOneBlockStore, stored...) + // one-block files are downloaded concurrently + var openedLock sync.Mutex var opened []string testOneBlockStore.OpenObjectFunc = func(ctx context.Context, name string) (io.ReadCloser, error) { + openedLock.Lock() + defer openedLock.Unlock() opened = append(opened, name) return io.NopCloser(bytes.NewReader(testOneBlockStore.Files[name])), nil } @@ -378,6 +383,8 @@ func TestForkableHub_LinkLiveUsingOneBlocks_StopsAtLiveBlock(t *testing.T) { require.False(t, fh.forkable.ForkDBHasLib()) require.NoError(t, fh.ProcessBlock(bstream.TestBlockWithLIBNum("00000005", "00000004", 4), nil)) + openedLock.Lock() + defer openedLock.Unlock() for _, name := range opened { num, err := strconv.ParseUint(name[:10], 10, 64) require.NoError(t, err) diff --git a/hub/subscription_test.go b/hub/subscription_test.go index 3055d97..aa5ff94 100644 --- a/hub/subscription_test.go +++ b/hub/subscription_test.go @@ -194,6 +194,13 @@ func TestSubscription_Run_WithPartialBlocks(t *testing.T) { handler := &testHandler{} sub := NewSubscription(handler, 10, true) + // Fill the channel before run() starts: which partial blocks it skips depends + // on what is already waiting behind them. + for _, block := range tt.blocks { + err := sub.push(block) + require.NoError(t, err) + } + // Start run() in a goroutine var runErr error var wg sync.WaitGroup @@ -203,12 +210,6 @@ func TestSubscription_Run_WithPartialBlocks(t *testing.T) { runErr = sub.run() }() - // Fill the channel with test blocks - for _, block := range tt.blocks { - err := sub.push(block) - require.NoError(t, err) - } - // Give some time for blocks to be processed time.Sleep(10 * time.Millisecond) From b23ed5c2ab27bd7dd858c239810fe4221860946c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 20:59:20 -0500 Subject: [PATCH 10/10] bumping dstore for azure 'startFrom' improvement --- go.mod | 104 +++++++++++++------------- go.sum | 226 +++++++++++++++++++++++++++++++-------------------------- 2 files changed, 179 insertions(+), 151 deletions(-) diff --git a/go.mod b/go.mod index 0cccd95..ffd73b2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/streamingfast/bstream -go 1.25.0 +go 1.26.0 require ( github.com/RoaringBitmap/roaring v1.9.4 @@ -10,7 +10,7 @@ require ( github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993 github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62 - github.com/streamingfast/dstore v0.2.3 + github.com/streamingfast/dstore v0.2.4-0.20261113015734-ed2936b371f8 github.com/streamingfast/logging v1.2.2 github.com/streamingfast/opaque v0.0.0-20210811180740-0c01d37ea308 github.com/streamingfast/pbgo v0.0.6-0.20251125204657-0a9c67563b19 @@ -28,42 +28,42 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.7.0 // indirect - cloud.google.com/go/logging v1.17.0 // indirect - cloud.google.com/go/longrunning v0.9.0 // indirect - cloud.google.com/go/monitoring v1.24.3 // indirect - cloud.google.com/go/storage v1.62.0 // indirect - cloud.google.com/go/trace v1.11.7 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/storage v1.69.0 // indirect + cloud.google.com/go/trace v1.16.0 // indirect connectrpc.com/connect v1.19.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/config v1.29.12 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.65 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect - github.com/aws/smithy-go v1.24.2 // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect + github.com/aws/smithy-go v1.28.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d // indirect @@ -78,16 +78,19 @@ require ( github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect - github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.26.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.20.1 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -95,6 +98,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openzipkin/zipkin-go v0.4.3 // indirect github.com/paulbellamy/ratecounter v0.2.0 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -104,9 +108,10 @@ require ( github.com/sercand/kuberesolver/v5 v5.1.1 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/streamingfast/sf-tracing v0.0.0-20251218140752-bafd5572499f // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect @@ -122,16 +127,17 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.54.0 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect + golang.org/x/net v0.59.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/term v0.45.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/api v0.278.0 // indirect - google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/api v0.288.0 // indirect + google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect ) diff --git a/go.sum b/go.sum index 6bd23c8..64923d1 100644 --- a/go.sum +++ b/go.sum @@ -12,93 +12,101 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= -cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= -cloud.google.com/go/logging v1.17.0 h1:rUFekZYwHiKElXCyz3zYBGz4BOeIqzgCKxVLdgrZ5mY= -cloud.google.com/go/logging v1.17.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= -cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= -cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= -cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= -cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= -cloud.google.com/go/storage v1.62.0 h1:w2pQJhpUqVerMON45vatE2FpCYsNTf7OHjkn6ux5mMU= -cloud.google.com/go/storage v1.62.0/go.mod h1:T5hz3qzcpnxZ5LdKc7y8Tw7lh4v9zeeVyrD/cLJAzZU= -cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= -cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q= +cloud.google.com/go/logging v1.19.0/go.mod h1:i40NZCHC9Gqvod4yE+yQfDWwlgwW/SrshkkGibCHxcA= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM= +cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= connectrpc.com/connect v1.19.2 h1:McQ83FGdzL+t60peksi0gXC7MQ/iLKgLduAnThbM0mo= connectrpc.com/connect v1.19.2/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= contrib.go.opencensus.io/exporter/stackdriver v0.12.6/go.mod h1:8x999/OcIPy5ivx/wDiV7Gx4D+VUPODf0mWRGRc5kSk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOrqXHhX1Kx/X/tCW9dEYQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 h1:yzIYdwuro811Z27D3T80Wkd3rqZzb0K43nner7Eh1yE= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 h1:5eCqTd9rTwMlE62z0xFdzPJ+3pji75hJrwq1jrCjo5w= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0/go.mod h1:4BcvJy7WxY8X2eX49z2VO1ByhO+CcQK8lKPCH/QlZvo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0 h1:mQdVn6c25/S2MHfJTWGSK3NwGoI/w9Ad7tzyLWbjAQI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0/go.mod h1:8W5IW/jylevlBQKSWkh5ZMP2oy7yT9Pnfug6Y6W/9D8= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/alecthomas/gometalinter v2.0.11+incompatible/go.mod h1:qfIpQGGz3d+NmgyPBqv+LSh50emm1pt72EtcX2vKYQk= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.29.12 h1:Y/2a+jLPrPbHpFkpAAYkVEtJmxORlXoo5k2g1fa2sUo= -github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.65 h1:q+nV2yYegofO/SUXruT+pn4KxkxmaQ++1B/QedcKBFM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 h1:iLdpkYZ4cXIQMO7ud+cqMWR1xK5ESbt1rvN77tRi1BY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43/go.mod h1:OgbsKPAswXDd5kxnR4vZov69p3oYjbvUyIRBAAV0y9o= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 h1:pdgODsAhGo4dvzC3JAG5Ce0PX8kWXrTZGx+jxADD+5E= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 h1:90uX0veLKcdHVfvxhkWUQSCi5VabtwMLFutYiRke4oo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= +github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 h1:wgxEej5cFj+EfutuAPZPIFcMvQ3Doamt01lMtPoMpls= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11/go.mod h1:dMcCQXtMtzVmEUO7YO+1xtYAvo8BcKgnN3Wppo8hbmA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= @@ -150,10 +158,12 @@ github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= @@ -168,6 +178,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -185,12 +197,12 @@ github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= -github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= -github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8= +github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= @@ -210,8 +222,10 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -244,6 +258,8 @@ github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7s github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -277,8 +293,8 @@ github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993 h1:gKU3/bj1DX5 github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993/go.mod h1:S6lJ9Dd6M3myzWbxspuhGggHtld9uYVN0xQ/7EGXmX8= github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62 h1:Tb4T34ImY7qvyrmW53/9ca5CxrXRThuFP1qx64i6FGk= github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62/go.mod h1:JbxEDbzWRG1dHdNIPrYfuPllEkktZMgm40AwVIBENcw= -github.com/streamingfast/dstore v0.2.3 h1:WQerHsINf2yT6Qj3mSwuxgWoDuiPkuqUolH0eQtnok0= -github.com/streamingfast/dstore v0.2.3/go.mod h1:O7Lsvvz0AnIJmW/CidKSBHbyW5xAK6AyD7L9kQ58Bbw= +github.com/streamingfast/dstore v0.2.4-0.20261113015734-ed2936b371f8 h1:ldOWWBRYaWflNpeEAP8H/SztvszPaHNDVn+1ihqHhbg= +github.com/streamingfast/dstore v0.2.4-0.20261113015734-ed2936b371f8/go.mod h1:DangUsbON/YBKPbxfUaoSFi9zupzHKP70q2bu9DhR68= github.com/streamingfast/logging v0.0.0-20210811175431-f3b44b61606a/go.mod h1:4GdqELhZOXj4xwc4IaBmzofzdErGynnaSzuzxy0ZIBo= github.com/streamingfast/logging v1.2.2 h1:hybYz/jtI+5PdGAAVLT+cGvSYo2Zi9dYpjTMRAvxjCQ= github.com/streamingfast/logging v1.2.2/go.mod h1:fJ5nP7ZSMB4MQQ6RM7cF+LiSQ43b5cVletcSUNL8z2M= @@ -304,15 +320,19 @@ github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmN github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= @@ -321,8 +341,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.41.0 h1:61oRQmYGMW7pXmFjPg1Muy84ndqMxQ6SH2L8fBG8fSY= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.41.0/go.mod h1:c0z2ubK4RQL+kSDuuFu9WnuXimObon3IiKjJf4NACvU= go.opentelemetry.io/otel/exporters/zipkin v1.45.0 h1:KN3btaILMTxR4QDHVGAO87lq5ButzK7l+kIfLuxQ1oA= @@ -363,10 +383,12 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -391,8 +413,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -405,8 +427,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -420,16 +442,16 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -460,8 +482,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E= -google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= +google.golang.org/api v0.288.0 h1:glhO/J88obKP5I269W3hB73dvBKrjU56ZfmNlNXpgTU= +google.golang.org/api v0.288.0/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -475,8 +497,8 @@ google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d h1:C9v1o0/4quuhOAfmRXA2j+we0PqZIp8traLdeogF3Ms= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d/go.mod h1:Wz2wFJntZFmLGo7pLDXZ3wYk5hyc0Mb+SkHhDDXT+lU= google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=