diff --git a/CHANGELOG.md b/CHANGELOG.md index 588dcad..d9810ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add `hub.WithMergedBlocksStore(store, bundleSize, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. +- A hub that is not ready because one-block files are missing between the blocks it loaded and the live blocks logs the missing range every 30s. It waits for those files instead of starting above them, so streams going from merged blocks to live blocks never skip blocks. +- Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `ForkableHub.WithoutPartials` and `stream.WithoutPartialBlocks`: stream from a hub fed with partial blocks without getting them, the last partial of each block arriving as the complete block, as for a relayer client that does not ask for partials. Lets an app that does not handle partial blocks share its hub with one that does. diff --git a/forkable/forkable.go b/forkable/forkable.go index a3cbbe4..d08d5cb 100644 --- a/forkable/forkable.go +++ b/forkable/forkable.go @@ -44,6 +44,10 @@ type Forkable struct { holdBlocksUntilLIB bool // if true, never passthrough anything before a LIB is set keptFinalBlocks int // how many blocks we keep behind LIB + // finalBlocksPurgeLimit, when set, can lower the block number under which final + // blocks are dropped, see WithFinalBlocksPurgeLimit. + finalBlocksPurgeLimit func(libNum, cutoff uint64) uint64 + includeInitialLIB bool failOnUnlinkableBlocksCount int @@ -860,7 +864,7 @@ func (p *Forkable) ProcessBlock(blk *pbbstream.Block, obj any) error { p.forkDB.MoveLIB(libRef) headBlockNum := newHeadBlock.Num() - _ = p.forkDB.PurgeBeforeLIBAndPartials(p.keptFinalBlocks, &headBlockNum) + _ = p.forkDB.PurgeBeforeLIBAndPartials(p.finalBlocksToKeep(libRef.Num()), &headBlockNum) if err := p.processIrreversibleSegment(irreversibleSegment, ppBlk.Block.AsRef()); err != nil { return err @@ -873,6 +877,25 @@ func (p *Forkable) ProcessBlock(blk *pbbstream.Block, obj any) error { return nil } +// finalBlocksToKeep is how many blocks are kept below a LIB at libNum: keptFinalBlocks, +// or more when finalBlocksPurgeLimit holds the purge back. +func (p *Forkable) finalBlocksToKeep(libNum uint64) int { + if p.finalBlocksPurgeLimit == nil { + return p.keptFinalBlocks + } + + var cutoff uint64 + if libNum > uint64(p.keptFinalBlocks) { + cutoff = libNum - uint64(p.keptFinalBlocks) + } + + limit := p.finalBlocksPurgeLimit(libNum, cutoff) + if limit >= cutoff { + return p.keptFinalBlocks + } + return int(libNum - limit) +} + func ids(blocks []*ForkableBlock) (ids []string) { ids = make([]string, len(blocks)) for i, obj := range blocks { diff --git a/forkable/options.go b/forkable/options.go index 6d7dacf..96dd837 100644 --- a/forkable/options.go +++ b/forkable/options.go @@ -104,6 +104,18 @@ func WithKeptFinalBlocks(count int) Option { } } +// WithFinalBlocksPurgeLimit lets the caller keep more final blocks than WithKeptFinalBlocks +// asks for. Each time the LIB moves, limit is called with the new LIB number and the +// cutoff the forkable is about to apply (blocks below it are dropped); returning a lower +// number keeps the blocks from that number on. A higher number is ignored. +// +// limit is called with the forkable locked, on the path of every block: it must not block. +func WithFinalBlocksPurgeLimit(limit func(libNum, cutoff uint64) uint64) Option { + return func(f *Forkable) { + f.finalBlocksPurgeLimit = limit + } +} + func EnsureBlockFlows(blockRef bstream.BlockRef) Option { return func(f *Forkable) { f.ensureBlockFlows = blockRef diff --git a/go.mod b/go.mod index 0cccd95..ffd73b2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/streamingfast/bstream -go 1.25.0 +go 1.26.0 require ( github.com/RoaringBitmap/roaring v1.9.4 @@ -10,7 +10,7 @@ require ( github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993 github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62 - github.com/streamingfast/dstore v0.2.3 + github.com/streamingfast/dstore v0.2.4-0.20261113015734-ed2936b371f8 github.com/streamingfast/logging v1.2.2 github.com/streamingfast/opaque v0.0.0-20210811180740-0c01d37ea308 github.com/streamingfast/pbgo v0.0.6-0.20251125204657-0a9c67563b19 @@ -28,42 +28,42 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.7.0 // indirect - cloud.google.com/go/logging v1.17.0 // indirect - cloud.google.com/go/longrunning v0.9.0 // indirect - cloud.google.com/go/monitoring v1.24.3 // indirect - cloud.google.com/go/storage v1.62.0 // indirect - cloud.google.com/go/trace v1.11.7 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/storage v1.69.0 // indirect + cloud.google.com/go/trace v1.16.0 // indirect connectrpc.com/connect v1.19.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/config v1.29.12 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.65 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect - github.com/aws/smithy-go v1.24.2 // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect + github.com/aws/smithy-go v1.28.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d // indirect @@ -78,16 +78,19 @@ require ( github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect - github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.26.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.20.1 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -95,6 +98,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openzipkin/zipkin-go v0.4.3 // indirect github.com/paulbellamy/ratecounter v0.2.0 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -104,9 +108,10 @@ require ( github.com/sercand/kuberesolver/v5 v5.1.1 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/streamingfast/sf-tracing v0.0.0-20251218140752-bafd5572499f // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect @@ -122,16 +127,17 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.54.0 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect + golang.org/x/net v0.59.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/term v0.45.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/api v0.278.0 // indirect - google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/api v0.288.0 // indirect + google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect ) diff --git a/go.sum b/go.sum index 6bd23c8..64923d1 100644 --- a/go.sum +++ b/go.sum @@ -12,93 +12,101 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= -cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= -cloud.google.com/go/logging v1.17.0 h1:rUFekZYwHiKElXCyz3zYBGz4BOeIqzgCKxVLdgrZ5mY= -cloud.google.com/go/logging v1.17.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= -cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= -cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= -cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= -cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= -cloud.google.com/go/storage v1.62.0 h1:w2pQJhpUqVerMON45vatE2FpCYsNTf7OHjkn6ux5mMU= -cloud.google.com/go/storage v1.62.0/go.mod h1:T5hz3qzcpnxZ5LdKc7y8Tw7lh4v9zeeVyrD/cLJAzZU= -cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= -cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q= +cloud.google.com/go/logging v1.19.0/go.mod h1:i40NZCHC9Gqvod4yE+yQfDWwlgwW/SrshkkGibCHxcA= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM= +cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= connectrpc.com/connect v1.19.2 h1:McQ83FGdzL+t60peksi0gXC7MQ/iLKgLduAnThbM0mo= connectrpc.com/connect v1.19.2/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= contrib.go.opencensus.io/exporter/stackdriver v0.12.6/go.mod h1:8x999/OcIPy5ivx/wDiV7Gx4D+VUPODf0mWRGRc5kSk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOrqXHhX1Kx/X/tCW9dEYQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 h1:yzIYdwuro811Z27D3T80Wkd3rqZzb0K43nner7Eh1yE= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 h1:5eCqTd9rTwMlE62z0xFdzPJ+3pji75hJrwq1jrCjo5w= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0/go.mod h1:4BcvJy7WxY8X2eX49z2VO1ByhO+CcQK8lKPCH/QlZvo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0 h1:mQdVn6c25/S2MHfJTWGSK3NwGoI/w9Ad7tzyLWbjAQI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0/go.mod h1:8W5IW/jylevlBQKSWkh5ZMP2oy7yT9Pnfug6Y6W/9D8= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/alecthomas/gometalinter v2.0.11+incompatible/go.mod h1:qfIpQGGz3d+NmgyPBqv+LSh50emm1pt72EtcX2vKYQk= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.29.12 h1:Y/2a+jLPrPbHpFkpAAYkVEtJmxORlXoo5k2g1fa2sUo= -github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.65 h1:q+nV2yYegofO/SUXruT+pn4KxkxmaQ++1B/QedcKBFM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 h1:iLdpkYZ4cXIQMO7ud+cqMWR1xK5ESbt1rvN77tRi1BY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43/go.mod h1:OgbsKPAswXDd5kxnR4vZov69p3oYjbvUyIRBAAV0y9o= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 h1:pdgODsAhGo4dvzC3JAG5Ce0PX8kWXrTZGx+jxADD+5E= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 h1:90uX0veLKcdHVfvxhkWUQSCi5VabtwMLFutYiRke4oo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= +github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 h1:wgxEej5cFj+EfutuAPZPIFcMvQ3Doamt01lMtPoMpls= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11/go.mod h1:dMcCQXtMtzVmEUO7YO+1xtYAvo8BcKgnN3Wppo8hbmA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= @@ -150,10 +158,12 @@ github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= @@ -168,6 +178,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -185,12 +197,12 @@ github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= -github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= -github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8= +github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= @@ -210,8 +222,10 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -244,6 +258,8 @@ github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7s github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -277,8 +293,8 @@ github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993 h1:gKU3/bj1DX5 github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993/go.mod h1:S6lJ9Dd6M3myzWbxspuhGggHtld9uYVN0xQ/7EGXmX8= github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62 h1:Tb4T34ImY7qvyrmW53/9ca5CxrXRThuFP1qx64i6FGk= github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62/go.mod h1:JbxEDbzWRG1dHdNIPrYfuPllEkktZMgm40AwVIBENcw= -github.com/streamingfast/dstore v0.2.3 h1:WQerHsINf2yT6Qj3mSwuxgWoDuiPkuqUolH0eQtnok0= -github.com/streamingfast/dstore v0.2.3/go.mod h1:O7Lsvvz0AnIJmW/CidKSBHbyW5xAK6AyD7L9kQ58Bbw= +github.com/streamingfast/dstore v0.2.4-0.20261113015734-ed2936b371f8 h1:ldOWWBRYaWflNpeEAP8H/SztvszPaHNDVn+1ihqHhbg= +github.com/streamingfast/dstore v0.2.4-0.20261113015734-ed2936b371f8/go.mod h1:DangUsbON/YBKPbxfUaoSFi9zupzHKP70q2bu9DhR68= github.com/streamingfast/logging v0.0.0-20210811175431-f3b44b61606a/go.mod h1:4GdqELhZOXj4xwc4IaBmzofzdErGynnaSzuzxy0ZIBo= github.com/streamingfast/logging v1.2.2 h1:hybYz/jtI+5PdGAAVLT+cGvSYo2Zi9dYpjTMRAvxjCQ= github.com/streamingfast/logging v1.2.2/go.mod h1:fJ5nP7ZSMB4MQQ6RM7cF+LiSQ43b5cVletcSUNL8z2M= @@ -304,15 +320,19 @@ github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmN github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= @@ -321,8 +341,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.41.0 h1:61oRQmYGMW7pXmFjPg1Muy84ndqMxQ6SH2L8fBG8fSY= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.41.0/go.mod h1:c0z2ubK4RQL+kSDuuFu9WnuXimObon3IiKjJf4NACvU= go.opentelemetry.io/otel/exporters/zipkin v1.45.0 h1:KN3btaILMTxR4QDHVGAO87lq5ButzK7l+kIfLuxQ1oA= @@ -363,10 +383,12 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -391,8 +413,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -405,8 +427,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -420,16 +442,16 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -460,8 +482,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E= -google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= +google.golang.org/api v0.288.0 h1:glhO/J88obKP5I269W3hB73dvBKrjU56ZfmNlNXpgTU= +google.golang.org/api v0.288.0/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -475,8 +497,8 @@ google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d h1:C9v1o0/4quuhOAfmRXA2j+we0PqZIp8traLdeogF3Ms= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d/go.mod h1:Wz2wFJntZFmLGo7pLDXZ3wYk5hyc0Mb+SkHhDDXT+lU= google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= diff --git a/hub/hub.go b/hub/hub.go index e22ffb8..5273279 100644 --- a/hub/hub.go +++ b/hub/hub.go @@ -59,7 +59,15 @@ type ForkableHub struct { oneBlockDownloadConcurrency int + mergedBlocks *mergedBlocksTracker // nil unless WithMergedBlocksStore is used + mergedBlocksLagMetric forkable.Uint64Metric + lastProgressLog time.Time + + // while not ready, since when live blocks do not link to the blocks loaded from + // one-block files, and when that was last logged + notLinkingSince time.Time + lastNotLinkingLog time.Time } const defaultOneBlockDownloadConcurrency = 32 @@ -105,6 +113,12 @@ func newForkableHub(liveSourceFactory bstream.SourceFactory, keepFinalBlocks int forkable.WithFilters(bstream.StepsAllWithPartial), }, extraForkableOptions...) + if hub.mergedBlocks != nil { + t := hub.mergedBlocks + t.maxKeptFinalBlocks = max(t.maxKeptFinalBlocks, 2*t.bundleSize, uint64(keepFinalBlocks)) + forkableOptions = append(forkableOptions, forkable.WithFinalBlocksPurgeLimit(hub.mergedBlocks.purgeLimit)) + } + hub.forkableOptions = forkableOptions hub.forkable = forkable.New(bstream.HandlerFunc(hub.broadcastBlock), forkableOptions...) @@ -407,7 +421,7 @@ func (h *ForkableHub) bootstrapOnce(ctx context.Context) error { if err != nil { return fmt.Errorf("parsing filename: %w", err) } - lowestBlockNum := substractAndRoundDownBlocks(refLibNum, uint64(h.keepFinalBlocks), bstream.DefaultMergedBlocksBundleSize) + lowestBlockNum := h.lowestBlockNumToLoad(ctx, refLibNum) filenames, err := h.unknownOneBlockFiles(sortedOneBlocksFiles, lowestBlockNum) if err != nil { @@ -440,6 +454,45 @@ func (h *ForkableHub) bootstrapOnce(ctx context.Context) error { return nil } +// lowestBlockNumToLoad is the block from which a hub that holds nothing yet loads one-block +// files, given the LIB of the most recent block it knows. +// +// A hub following the merged-blocks store starts at its last file, so it covers every +// block the files do not hold. Otherwise it starts keepFinalBlocks below LIB, rounded down +// to a merged-blocks file boundary. +// +// It runs for every live block the hub cannot link until it has a LIB, so the store is +// only searched until a search succeeds, and at most once per mergedBlocksPollInterval +// while searches fail. +func (h *ForkableHub) lowestBlockNumToLoad(ctx context.Context, libNum uint64) uint64 { + if h.mergedBlocks == nil { + bundleSize := bstream.SanitizeBundleSize(bstream.DefaultMergedBlocksBundleSize) + return substractAndRoundDownBlocks(libNum, uint64(h.keepFinalBlocks), bundleSize) + } + + t := h.mergedBlocks + if start, ok := t.knownStart(libNum); ok { + return start + } + + ctx, cancel := context.WithTimeout(ctx, mergedBlocksLookupTimeout) + defer cancel() + + // two files below LIB is where the last merged-blocks file is when the merger keeps up + assumedStart := substractAndRoundDownBlocks(libNum, 2*t.bundleSize, t.bundleSize) + if t.lookupFailedRecently() { + return assumedStart + } + + located, err := t.locate(ctx, libNum) + if err != nil { + t.lookupFailedAt.Store(time.Now().UnixNano()) + h.logger.Warn("cannot find the last merged-blocks file, assuming the merger keeps up", zap.Error(err)) + return assumedStart + } + return located +} + func (h *ForkableHub) Run() { liveSource := h.liveSourceFactory(h) liveSource.OnTerminating(h.reconnect) @@ -452,6 +505,10 @@ func (h *ForkableHub) Run() { close(h.Ready) } + if h.mergedBlocks != nil { + go h.trackMergedBlocks() + } + liveSource.Run() } @@ -500,6 +557,9 @@ func (h *ForkableHub) ProcessBlock(blk *pbbstream.Block, obj any) error { // limit would be reached after a quarter of the blocks it names — and the check is // there to catch a gap the one-block store can no longer bridge, which the block's // final partial reports just as well. A real block and a final flash block count. + if !h.IsReady() { + h.logNotLinking(blk) + } if h.maxConsecutiveUnlinkableBlocks != 0 && !isNonFinalPartial(blk) { h.consecutiveUnlinkableBlocks++ h.logger.Warn("block not linkable after one-block lookup", @@ -513,6 +573,7 @@ func (h *ForkableHub) ProcessBlock(blk *pbbstream.Block, obj any) error { } } else { // linkable h.consecutiveUnlinkableBlocks = 0 + h.notLinkingSince = time.Time{} if !h.IsReady() { h.logger.Info("Hub is ready") close(h.Ready) @@ -526,11 +587,57 @@ func (h *ForkableHub) ProcessBlock(blk *pbbstream.Block, obj any) error { var errRestartRequired = errors.New("restart required") +// notLinkingLogInterval is how long live blocks must fail to link to the blocks a hub +// loaded from one-block files before it says so, and how often it says it again. +var notLinkingLogInterval = 30 * time.Second + +// logNotLinking explains why a hub that loaded blocks from one-block files is not ready: +// the live blocks do not link to them, because one-block files are missing in between. +// The hub does not skip over the missing blocks, since streams going from merged blocks +// to live blocks would miss them; it gets ready once the files show up. +func (h *ForkableHub) logNotLinking(blk *pbbstream.Block) { + if !h.forkable.ForkDBHasLib() { + // still bootstrapping from live blocks + return + } + + now := time.Now() + if h.notLinkingSince.IsZero() { + h.notLinkingSince = now + } + if now.Sub(h.notLinkingSince) < notLinkingLogInterval || now.Sub(h.lastNotLinkingLog) < notLinkingLogInterval { + return + } + h.lastNotLinkingLog = now + + // the lowest block the live blocks reach by following parent links + lowestLive := blk + for { + parent := h.forkable.GetBlockByHash(lowestLive.ParentId) + if parent == nil { + break + } + lowestLive = parent + } + + h.logger.Warn("hub not ready: one-block files are missing between the blocks loaded from one-block files and the live blocks, waiting for them to show up", + zap.Uint64("missing_from_block", h.forkable.HeadNum()+1), + zap.Uint64("missing_up_to_block", lowestLive.Number-1), + zap.Uint64("lib_num", h.forkable.LIBNum()), + zap.Uint64("live_block_num", blk.Number), + zap.Duration("not_linking_for", now.Sub(h.notLinkingSince)), + ) +} + func (h *ForkableHub) linkLiveUsingOneBlocks(ctx context.Context, blk *pbbstream.Block) error { lastKnownLib := h.forkable.LowestBlockNum() if !h.forkable.ForkDBHasLib() { lastKnownLib = blk.LibNum + if h.mergedBlocks != nil { + // same starting point as bootstrap + lastKnownLib = min(lastKnownLib, h.lowestBlockNumToLoad(ctx, blk.LibNum)) + } } h.logger.Info("linking live block using one blocks", zap.Uint64("processed_block", blk.Number), zap.Uint64("last_know_lib", lastKnownLib)) diff --git a/hub/hub_test.go b/hub/hub_test.go index 09bd58f..923f1d6 100644 --- a/hub/hub_test.go +++ b/hub/hub_test.go @@ -8,6 +8,7 @@ import ( "io" "strconv" "strings" + "sync" "testing" "time" @@ -369,8 +370,12 @@ func TestForkableHub_LinkLiveUsingOneBlocks_StopsAtLiveBlock(t *testing.T) { } AddToMockStore(t, testOneBlockStore, stored...) + // one-block files are downloaded concurrently + var openedLock sync.Mutex var opened []string testOneBlockStore.OpenObjectFunc = func(ctx context.Context, name string) (io.ReadCloser, error) { + openedLock.Lock() + defer openedLock.Unlock() opened = append(opened, name) return io.NopCloser(bytes.NewReader(testOneBlockStore.Files[name])), nil } @@ -378,6 +383,8 @@ func TestForkableHub_LinkLiveUsingOneBlocks_StopsAtLiveBlock(t *testing.T) { require.False(t, fh.forkable.ForkDBHasLib()) require.NoError(t, fh.ProcessBlock(bstream.TestBlockWithLIBNum("00000005", "00000004", 4), nil)) + openedLock.Lock() + defer openedLock.Unlock() for _, name := range opened { num, err := strconv.ParseUint(name[:10], 10, 64) require.NoError(t, err) diff --git a/hub/merged_blocks.go b/hub/merged_blocks.go new file mode 100644 index 0000000..2645836 --- /dev/null +++ b/hub/merged_blocks.go @@ -0,0 +1,364 @@ +package hub + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/streamingfast/bstream" + "github.com/streamingfast/bstream/forkable" + "github.com/streamingfast/dstore" + "go.uber.org/zap" +) + +// mergedBlocksPollInterval is how often the hub checks whether the next merged-blocks file +// showed up. It is a var so tests can shrink it. +var mergedBlocksPollInterval = 5 * time.Second + +// mergedBlocksLookupTimeout bounds each search of the merged-blocks store, so a store call +// that hangs does not stop the hub from following the merger. It is a var so tests can +// shrink it. +var mergedBlocksLookupTimeout = 30 * time.Second + +// mergedBlocksBehindLogInterval is how often the hub warns that it keeps extra final +// blocks because merged blocks are behind. +const mergedBlocksBehindLogInterval = 30 * time.Second + +// WithMergedBlocksStore makes the hub keep its final blocks until they are available in a +// merged-blocks file of store: it holds every block from the start of the last +// merged-blocks file it found, so a stream reading the files can always continue on the +// hub. The hub is then built with a keepFinalBlocks of 0, which only matters to a hub +// without this option: a fixed depth below LIB leaves, when the merger falls further +// behind than that depth, a range of blocks that neither the files nor the hub can serve. +// +// The hub finds where merged blocks end with one listing of the store when it starts. It +// then learns about new files with one FileExists call on the next expected file, at most +// every 5 seconds and only once LIB passed the end of that file, or with one listing when +// LIB is far enough for several new files to be there, as when the merger catches up. +// +// bundleSize is the number of blocks per merged-blocks file in store (0 means +// bstream.DefaultMergedBlocksBundleSize). +// +// maxKeptFinalBlocks limits the final blocks held in memory when the merger is behind or +// the store cannot be reached. Past it the oldest blocks are dropped even though no +// merged-blocks file holds them. It must cover at least two merged-blocks files, which is +// what the hub holds when the merger keeps up. +func WithMergedBlocksStore(store dstore.Store, bundleSize uint64, maxKeptFinalBlocks int) Option { + return func(h *ForkableHub) { + h.mergedBlocks = &mergedBlocksTracker{ + store: store, + bundleSize: bstream.SanitizeBundleSize(bundleSize), + maxKeptFinalBlocks: uint64(maxKeptFinalBlocks), + } + } +} + +// WithMergedBlocksLagMetric reports how many blocks LIB is ahead of the last block found +// in the merged-blocks store. It needs WithMergedBlocksStore. With a merger keeping up, +// the value moves between 1 and a little over one merged-blocks file; it keeps growing +// when the merger is stopped or too slow. The hub does not look for merged-blocks files +// more than maxKeptFinalBlocks below LIB, so a hub starting while the merger is further +// behind reports about maxKeptFinalBlocks at first. +func WithMergedBlocksLagMetric(metric forkable.Uint64Metric) Option { + return func(h *ForkableHub) { + h.mergedBlocksLagMetric = metric + } +} + +// mergedBlocksTracker follows the merged-blocks store from the hub's point of view: which +// is the first file, going up, that is not there yet. +type mergedBlocksTracker struct { + store dstore.Store + bundleSize uint64 + maxKeptFinalBlocks uint64 + + lookupLock sync.Mutex // serializes the store lookups + + // nextBase is the base block number of the first merged-blocks file not seen in the + // store. Every file below it was seen, or sits under where the hub started looking: + // the hub never looks more than maxKeptFinalBlocks below LIB. + nextBase atomic.Uint64 + known atomic.Bool + + // lookupFailedAt is when the hub last failed to locate the merged blocks while + // starting, in Unix nanoseconds. + lookupFailedAt atomic.Int64 +} + +func mergedBlocksFilename(base uint64) string { + return fmt.Sprintf("%010d", base) +} + +// fileExists tells whether the merged-blocks file starting at base is in the store. A +// lookup that fails is retried like a bootstrap attempt is: bootstrapMaxAttempts times, +// bootstrapRetryDelay apart. A missing file is an answer, not a failure. +func (t *mergedBlocksTracker) fileExists(ctx context.Context, base uint64) (exists bool, err error) { + for attempt := range bootstrapMaxAttempts { + if attempt > 0 { + select { + case <-ctx.Done(): + return false, fmt.Errorf("looking up merged-blocks file %s: %w", mergedBlocksFilename(base), ctx.Err()) + case <-time.After(bootstrapRetryDelay): + } + } + if exists, err = t.store.FileExists(ctx, mergedBlocksFilename(base)); err == nil { + return exists, nil + } + } + return false, fmt.Errorf("looking up merged-blocks file %s: %w", mergedBlocksFilename(base), err) +} + +// firstRunFrom lists the store from the merged-blocks file starting at base, and returns +// the first run of consecutive files it holds there: the base of its first file and the +// base that follows its last one. ok is false when the store holds no file from base on. +// +// It is one listing whatever the number of files, stopped at the end of the run. A listing +// that fails is retried like a file lookup is. +func (t *mergedBlocksTracker) firstRunFrom(ctx context.Context, base uint64) (first, next uint64, ok bool, err error) { + for attempt := range bootstrapMaxAttempts { + if attempt > 0 { + select { + case <-ctx.Done(): + return 0, 0, false, fmt.Errorf("listing merged-blocks files from %s: %w", mergedBlocksFilename(base), ctx.Err()) + case <-time.After(bootstrapRetryDelay): + } + } + + ok = false + err = t.store.WalkFrom(ctx, "", mergedBlocksFilename(base), func(filename string) error { + name, _, _ := strings.Cut(filename, ".") + fileBase, parseErr := strconv.ParseUint(name, 10, 64) + if parseErr != nil || fileBase%t.bundleSize != 0 || fileBase < base { + return nil // not a merged-blocks file of this store's size + } + if !ok { + first, next, ok = fileBase, fileBase+t.bundleSize, true + return nil + } + if fileBase != next { + return dstore.StopIteration + } + next += t.bundleSize + return nil + }) + if err == nil { + return first, next, ok, nil + } + } + return 0, 0, false, fmt.Errorf("listing merged-blocks files from %s: %w", mergedBlocksFilename(base), err) +} + +// locate finds where the merged blocks end and returns the block the hub must start from +// at a LIB of libNum for its blocks to overlap the last merged-blocks file. That block is +// never further than maxKeptFinalBlocks below libNum, nor below the first streamable +// block, and neither is where locate looks. +// +// It takes the first run of consecutive files in that range, so a file the merger wrote +// ahead of a lower one that is still missing is not taken for the end of merged blocks. +func (t *mergedBlocksTracker) locate(ctx context.Context, libNum uint64) (uint64, error) { + t.lookupLock.Lock() + defer t.lookupLock.Unlock() + + // the lowest file the hub can start from + floor := t.firstBase() + if reach := libNum + t.bundleSize; reach > t.maxKeptFinalBlocks { + floor = max(floor, (reach-t.maxKeptFinalBlocks-1)/t.bundleSize*t.bundleSize) + } + + _, next, ok, err := t.firstRunFrom(ctx, floor) + if err != nil { + return 0, err + } + + // Without a file in reach, either there is none yet, or the merged blocks end lower + // than the hub can go: the merger writes files in order, so it writes the one at floor + // once it gets there. + start, nextBase := floor, floor + if ok { + start, nextBase = next-t.bundleSize, next + } + + t.nextBase.Store(nextBase) + t.known.Store(true) + + return max(start, bstream.GetProtocolFirstStreamableBlock), nil +} + +// knownStart is the block the hub must start from at a LIB of libNum, computed from what +// the last successful locate found instead of searching the store: the start of the last +// merged-blocks file, never further than maxKeptFinalBlocks below libNum, nor below the +// first streamable block. ok is false when no locate succeeded yet. +func (t *mergedBlocksTracker) knownStart(libNum uint64) (start uint64, ok bool) { + if !t.known.Load() { + return 0, false + } + + start = t.firstBase() + if nextBase := t.nextBase.Load(); nextBase >= start+t.bundleSize { + start = nextBase - t.bundleSize + } + if libNum > t.maxKeptFinalBlocks { + start = max(start, libNum-t.maxKeptFinalBlocks) + } + return max(start, bstream.GetProtocolFirstStreamableBlock), true +} + +// lookupFailedRecently reports whether locating the merged blocks failed less than +// mergedBlocksPollInterval ago. +func (t *mergedBlocksTracker) lookupFailedRecently() bool { + failedAt := t.lookupFailedAt.Load() + return failedAt != 0 && time.Since(time.Unix(0, failedAt)) < mergedBlocksPollInterval +} + +// firstBase is the base of the merged-blocks file holding the first streamable block. +func (t *mergedBlocksTracker) firstBase() uint64 { + return bstream.GetProtocolFirstStreamableBlock / t.bundleSize * t.bundleSize +} + +// advance moves nextBase over the files that showed up. A file is only looked up once LIB +// passed its last block, since the merger cannot have written it before. +func (t *mergedBlocksTracker) advance(ctx context.Context, libNum uint64) error { + t.lookupLock.Lock() + defer t.lookupLock.Unlock() + + base := t.nextBase.Load() + if libNum < base+t.bundleSize { + return nil + } + + // only the next file can be there: one lookup + if libNum < base+2*t.bundleSize { + exists, err := t.fileExists(ctx, base) + if err != nil { + return err + } + if exists { + t.nextBase.Store(base + t.bundleSize) + } + return nil + } + + // several files can be there, as when the merger catches up: one listing + first, next, ok, err := t.firstRunFrom(ctx, base) + if err != nil { + return err + } + if ok && first == base { + t.nextBase.Store(next) + } + return nil +} + +// lag is how many blocks libNum is ahead of the last merged block: the final blocks that +// are not in a merged-blocks file. +func (t *mergedBlocksTracker) lag(libNum uint64) uint64 { + // before the first merged-blocks file, the lag counts from the first streamable block + firstUnmerged := max(t.nextBase.Load(), bstream.GetProtocolFirstStreamableBlock) + if libNum < firstUnmerged { + return 0 + } + return libNum + 1 - firstUnmerged +} + +// purgeLimit is a forkable.WithFinalBlocksPurgeLimit function. It keeps the blocks of the +// last merged-blocks file and everything above: a stream reading the files joins the hub +// on a block it received from a file, so the hub has to overlap the last one. +func (t *mergedBlocksTracker) purgeLimit(libNum, cutoff uint64) uint64 { + // Nothing is dropped on account of merged blocks until the hub knows where they end. + var limit uint64 + if nextBase := t.nextBase.Load(); t.known.Load() && nextBase > t.bundleSize { + limit = nextBase - t.bundleSize + } + if libNum > t.maxKeptFinalBlocks { + limit = max(limit, libNum-t.maxKeptFinalBlocks) + } + return min(limit, cutoff) +} + +// MergedBlocksUpTo returns the first block number the hub has not found in a merged-blocks +// file, at most maxKeptFinalBlocks below LIB when the hub started: the files end there or +// lower. ok is false when the hub does not follow that store (see WithMergedBlocksStore) +// or has not looked yet. +func (h *ForkableHub) MergedBlocksUpTo() (blockNum uint64, ok bool) { + if h == nil || h.mergedBlocks == nil || !h.mergedBlocks.known.Load() { + return 0, false + } + return h.mergedBlocks.nextBase.Load(), true +} + +// followMergedBlocks locates the merged blocks if no search succeeded yet, then moves +// over the files that showed up. It returns false when the merged blocks are not located. +func (h *ForkableHub) followMergedBlocks(ctx context.Context, libNum uint64) bool { + ctx, cancel := context.WithTimeout(ctx, mergedBlocksLookupTimeout) + defer cancel() + + t := h.mergedBlocks + if !t.known.Load() { + if _, err := t.locate(ctx, libNum); err != nil { + h.logger.Warn("cannot find the last merged-blocks file", zap.Error(err)) + return false + } + } + + if err := t.advance(ctx, libNum); err != nil { + h.logger.Warn("cannot check for new merged-blocks files", zap.Error(err)) + } + return true +} + +// trackMergedBlocks keeps the merged blocks tracker and the lag metric up to date until +// the hub terminates. +func (h *ForkableHub) trackMergedBlocks() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-h.Terminating() + cancel() + }() + + ticker := time.NewTicker(mergedBlocksPollInterval) + defer ticker.Stop() + + var lastBehindLog time.Time + for { + select { + case <-h.Terminating(): + return + case <-ticker.C: + } + + if !h.IsReady() { + continue + } + + t := h.mergedBlocks + libNum := h.forkable.LIBNum() + + if !h.followMergedBlocks(ctx, libNum) { + continue + } + + lag := t.lag(libNum) + if h.mergedBlocksLagMetric != nil { + h.mergedBlocksLagMetric.SetUint64(lag) + } + + // a merger keeping up is at most one file behind LIB, plus the time it takes to write it + if lag > 3*t.bundleSize && time.Since(lastBehindLog) >= mergedBlocksBehindLogInterval { + lastBehindLog = time.Now() + lowest := h.forkable.LowestBlockNum() + h.logger.Warn("merged blocks are behind, keeping more final blocks in memory to cover for them", + zap.Uint64("lib_num", libNum), + zap.Uint64("next_merged_blocks_file", t.nextBase.Load()), + zap.Uint64("merged_blocks_lag", lag), + zap.Uint64("lowest_block_num", lowest), + zap.Uint64("max_kept_final_blocks", t.maxKeptFinalBlocks), + zap.Bool("blocks_missing_between_merged_and_hub", lowest > t.nextBase.Load()), + ) + } + } +} diff --git a/hub/merged_blocks_test.go b/hub/merged_blocks_test.go new file mode 100644 index 0000000..0d2b01a --- /dev/null +++ b/hub/merged_blocks_test.go @@ -0,0 +1,689 @@ +package hub + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/streamingfast/bstream" + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + "github.com/streamingfast/dstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +// storeObserver records the calls the hub makes on a merged-blocks store, as +// "exists " for a lookup and "list " for a listing from that file, and makes +// them fail while failing is set. +type storeObserver struct { + sync.Mutex + calls []string + failing bool +} + +func observeStore(store *dstore.MockStore) *storeObserver { + o := &storeObserver{} + record := func(call string) error { + o.Lock() + defer o.Unlock() + o.calls = append(o.calls, call) + if o.failing { + return errors.New("store unavailable") + } + return nil + } + + store.FileExistsFunc = func(_ context.Context, base string) (bool, error) { + if err := record("exists " + base); err != nil { + return false, err + } + _, found := store.Files[base] + return found, nil + } + store.WalkFromFunc = func(ctx context.Context, prefix, startingPoint string, f func(filename string) error) error { + if err := record("list " + startingPoint); err != nil { + return err + } + return listMockStoreFrom(store, startingPoint, f) + } + return o +} + +// listMockStoreFrom is what MockStore.WalkFrom does without a WalkFromFunc. +func listMockStoreFrom(store *dstore.MockStore, startingPoint string, f func(filename string) error) error { + var names []string + for name := range store.Files { + if name >= startingPoint { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + if err := f(name); err != nil { + if errors.Is(err, dstore.StopIteration) { + return nil + } + return err + } + } + return nil +} + +func (o *storeObserver) setFailing(failing bool) { + o.Lock() + defer o.Unlock() + o.failing = failing +} + +func (o *storeObserver) seen() []string { + o.Lock() + defer o.Unlock() + return append([]string(nil), o.calls...) +} + +// testChain returns blocks from..to, each one final two blocks later. +func testChain(from, to uint64) (out []*pbbstream.Block) { + id := func(num uint64) string { return fmt.Sprintf("%08x", num) } + for num := from; num <= to; num++ { + out = append(out, bstream.TestBlockWithLIBNum(id(num), id(num-1), num-2)) + } + return out +} + +func setMergedBlocksFiles(store *dstore.MockStore, bases ...uint64) { + for _, base := range bases { + store.SetFile(mergedBlocksFilename(base), []byte("merged")) + } +} + +func newMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBases []uint64, maxKeptFinalBlocks int) (*ForkableHub, *dstore.MockStore) { + t.Helper() + + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + oneBlocksStore := dstore.NewMockStore(nil) + AddToMockStore(t, oneBlocksStore, oneBlocks...) + + mergedBlocksStore := dstore.NewMockStore(nil) + setMergedBlocksFiles(mergedBlocksStore, mergedBases...) + + lsf := bstream.NewTestSourceFactory() + fh := NewForkableHubWithOptions(lsf.NewSource, 0, oneBlocksStore, []Option{WithMergedBlocksStore(mergedBlocksStore, 100, maxKeptFinalBlocks)}) + return fh, mergedBlocksStore +} + +func processBlocks(t *testing.T, fh *ForkableHub, blocks []*pbbstream.Block) { + t.Helper() + for _, blk := range blocks { + require.NoError(t, fh.ProcessBlock(blk, nil)) + } +} + +func TestForkableHub_KeepsFinalBlocksUntilMerged(t *testing.T) { + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{100, 200, 300}, 2000) + require.NoError(t, fh.bootstrap()) + require.Equal(t, uint64(300), fh.forkable.LowestBlockNum()) + + // LIB reaches 898 while the last merged-blocks file still ends at 399 + processBlocks(t, fh, testChain(511, 900)) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "blocks of the last merged-blocks file and above are kept") + assert.Equal(t, uint64(499), fh.mergedBlocks.lag(fh.forkable.LIBNum())) + + var fromLastMergedFile []*bstream.PreprocessedBlock + require.NoError(t, fh.forkable.CallWithBlocksFromNum(399, func(blocks []*bstream.PreprocessedBlock) { fromLastMergedFile = blocks }, false)) + require.NotEmpty(t, fromLastMergedFile) + assert.Equal(t, uint64(399), fromLastMergedFile[0].Block.Number) + + // the merger catches up + setMergedBlocksFiles(mergedBlocksStore, 400, 500, 600) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(901, 901)) + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "blocks below the last merged-blocks file are dropped") + assert.Equal(t, uint64(200), fh.mergedBlocks.lag(fh.forkable.LIBNum())) + + // the hub follows the merged-blocks files as they show up + setMergedBlocksFiles(mergedBlocksStore, 700) + processBlocks(t, fh, testChain(902, 1010)) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(1011, 1011)) + assert.Equal(t, uint64(700), fh.forkable.LowestBlockNum()) + + setMergedBlocksFiles(mergedBlocksStore, 800, 900) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(1012, 1012)) + assert.Equal(t, uint64(900), fh.forkable.LowestBlockNum()) +} + +func TestForkableHub_MergedBlocksBundleSize(t *testing.T) { + require.Equal(t, uint64(100), bstream.DefaultMergedBlocksBundleSize) + + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + oneBlocksStore := dstore.NewMockStore(nil) + AddToMockStore(t, oneBlocksStore, testChain(2000, 3100)...) + mergedBlocksStore := dstore.NewMockStore(nil) + setMergedBlocksFiles(mergedBlocksStore, 0, 1000, 2000) + + fh := NewForkableHubWithOptions(bstream.NewTestSourceFactory().NewSource, 0, oneBlocksStore, []Option{WithMergedBlocksStore(mergedBlocksStore, 1000, 20000)}) + require.NoError(t, fh.bootstrap()) + assert.Equal(t, uint64(2000), fh.forkable.LowestBlockNum()) + upTo, ok := fh.MergedBlocksUpTo() + require.True(t, ok) + assert.Equal(t, uint64(3000), upTo) + + setMergedBlocksFiles(mergedBlocksStore, 3000) + processBlocks(t, fh, testChain(3101, 4010)) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), fh.forkable.LIBNum())) + processBlocks(t, fh, testChain(4011, 4011)) + + upTo, _ = fh.MergedBlocksUpTo() + assert.Equal(t, uint64(4000), upTo) + assert.Equal(t, uint64(3000), fh.forkable.LowestBlockNum()) +} + +func TestForkableHub_MaxKeptFinalBlocks(t *testing.T) { + fh, _ := newMergedBlocksTestHub(t, testChain(300, 510), []uint64{300}, 350) + require.NoError(t, fh.bootstrap()) + + processBlocks(t, fh, testChain(511, 900)) + assert.Equal(t, uint64(898-350), fh.forkable.LowestBlockNum()) +} + +func TestForkableHub_BootstrapFromLastMergedBlocksFile(t *testing.T) { + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) + require.NoError(t, fh.bootstrap()) + + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "starts from the last merged-blocks file") + assert.Equal(t, uint64(400), fh.mergedBlocks.nextBase.Load()) +} + +func TestForkableHub_BootstrapWithMergedBlocksUpToDate(t *testing.T) { + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300, 400, 500, 600, 700}, 2000) + store := observeStore(mergedBlocksStore) + + require.NoError(t, fh.bootstrap()) + assert.Equal(t, uint64(700), fh.forkable.LowestBlockNum()) + assert.Equal(t, []string{"list 0000000000"}, store.seen(), "one listing whatever the number of files") + + // nothing until LIB passes the end of the next file + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 899)) + assert.Equal(t, []string{"list 0000000000"}, store.seen()) + + // one lookup while only the next file can be there + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 900)) + assert.Equal(t, []string{"list 0000000000", "exists 0000000800"}, store.seen()) + assert.Equal(t, uint64(800), fh.mergedBlocks.nextBase.Load()) + + // one listing once several files can be there + setMergedBlocksFiles(mergedBlocksStore, 800, 900, 1000) + require.NoError(t, fh.mergedBlocks.advance(context.Background(), 1150)) + assert.Equal(t, []string{"list 0000000000", "exists 0000000800", "list 0000000800"}, store.seen()) + assert.Equal(t, uint64(1100), fh.mergedBlocks.nextBase.Load()) +} + +func TestMergedBlocksTracker_LocateStopsAtMaxKeptFinalBlocks(t *testing.T) { + bases := func(from, to uint64) (out []uint64) { + for base := from; base <= to; base += 100 { + out = append(out, base) + } + return out + } + + tests := []struct { + name string + firstStreamableBlock uint64 + mergedBases []uint64 + expectedLowest uint64 + expectedNextBase uint64 + }{ + {"no merged-blocks file at all", 0, nil, 4600, 4600}, + {"no merged-blocks file at all, chain starting above 0", 1250, nil, 4600, 4600}, + {"only the first file", 0, bases(0, 0), 4600, 4600}, + {"merged blocks end far below", 0, bases(0, 1200), 4600, 4600}, + {"merged blocks end far below, chain starting above 0", 1250, bases(1200, 3300), 4600, 4600}, + {"merged blocks end one file below reach", 0, bases(0, 4500), 4600, 4600}, + {"merged blocks end within reach", 0, bases(0, 4600), 4600, 4700}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = test.firstStreamableBlock + defer func() { bstream.GetProtocolFirstStreamableBlock = prev }() + + store := dstore.NewMockStore(nil) + setMergedBlocksFiles(store, test.mergedBases...) + observed := observeStore(store) + + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 500} + + lowest, err := tracker.locate(context.Background(), 5010) + require.NoError(t, err) + assert.Equal(t, test.expectedLowest, lowest) + assert.Equal(t, test.expectedNextBase, tracker.nextBase.Load()) + assert.Len(t, observed.seen(), 1, "one listing, from maxKeptFinalBlocks below LIB") + }) + } +} + +func TestMergedBlocksTracker_BeforeFirstMergedBlocksFile(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 150 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + store := dstore.NewMockStore(nil) + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} + + lowest, err := tracker.locate(context.Background(), 180) + require.NoError(t, err) + assert.Equal(t, uint64(150), lowest, "never below the first streamable block") + assert.Equal(t, uint64(100), tracker.nextBase.Load()) + assert.Equal(t, uint64(31), tracker.lag(180), "counted from the first streamable block") + assert.Equal(t, uint64(0), tracker.purgeLimit(450, 450), "everything is kept until the first file exists") + + // the first file holds blocks 150 to 199 + setMergedBlocksFiles(store, 100) + require.NoError(t, tracker.advance(context.Background(), 199)) + assert.Equal(t, uint64(100), tracker.nextBase.Load(), "not looked up before LIB passed its end") + require.NoError(t, tracker.advance(context.Background(), 205)) + assert.Equal(t, uint64(200), tracker.nextBase.Load()) + assert.Equal(t, uint64(6), tracker.lag(205)) +} + +func TestForkableHub_LiveBootstrapFromLastMergedBlocksFile(t *testing.T) { + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) + + // no bootstrap from one-block files: the hub starts from its first live block + processBlocks(t, fh, testChain(901, 901)) + + require.True(t, fh.IsReady()) + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum(), "starts from the last merged-blocks file instead of the live block's LIB") + assert.Equal(t, uint64(400), fh.mergedBlocks.nextBase.Load()) +} + +func TestMergedBlocksTracker_RetriesFailedLookups(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + t.Cleanup(func() { bootstrapRetryDelay = prev }) + + store := dstore.NewMockStore(nil) + setMergedBlocksFiles(store, 300) + + failures := 0 + store.WalkFromFunc = func(_ context.Context, _, startingPoint string, f func(filename string) error) error { + if failures > 0 { + failures-- + return errors.New("store unavailable") + } + return listMockStoreFrom(store, startingPoint, f) + } + + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} + + failures = bootstrapMaxAttempts - 1 + lowest, err := tracker.locate(context.Background(), 800) + require.NoError(t, err) + assert.Equal(t, uint64(300), lowest) + + failures = bootstrapMaxAttempts + _, err = tracker.locate(context.Background(), 800) + require.ErrorContains(t, err, "store unavailable") +} + +func TestForkableHub_MergedBlocksUpTo(t *testing.T) { + var nilHub *ForkableHub + _, ok := nilHub.MergedBlocksUpTo() + assert.False(t, ok) + + fh, _ := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) + _, ok = fh.MergedBlocksUpTo() + assert.False(t, ok, "not known before the hub looked") + + require.NoError(t, fh.bootstrap()) + upTo, ok := fh.MergedBlocksUpTo() + assert.True(t, ok) + assert.Equal(t, uint64(400), upTo) +} + +func TestMergedBlocksTracker_PurgeLimit(t *testing.T) { + tracker := &mergedBlocksTracker{bundleSize: 100, maxKeptFinalBlocks: 2000} + + assert.Equal(t, uint64(0), tracker.purgeLimit(1500, 1500), "nothing dropped before the merged blocks are located") + assert.Equal(t, uint64(3000), tracker.purgeLimit(5000, 5000), "except past maxKeptFinalBlocks") + + tracker.nextBase.Store(4800) + tracker.known.Store(true) + assert.Equal(t, uint64(4700), tracker.purgeLimit(5000, 5000), "keeps the last merged-blocks file") + assert.Equal(t, uint64(4600), tracker.purgeLimit(5000, 4600), "never above the forkable's own cutoff") +} + +func TestForkableHub_BootstrapWhenMergedBlocksStoreFails(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + t.Cleanup(func() { bootstrapRetryDelay = prev }) + + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) + observeStore(mergedBlocksStore).setFailing(true) + + require.NoError(t, fh.bootstrap()) + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "two merged-blocks files below LIB") + + processBlocks(t, fh, testChain(901, 1200)) + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "nothing is dropped while the merged blocks are not located") +} + +// liveChainWithoutLIB returns blocks from..to whose LIB is 100 blocks below them, so a hub +// receiving only these blocks has no LIB in its forkDB. +func liveChainWithoutLIB(from, to uint64) (out []*pbbstream.Block) { + id := func(num uint64) string { return fmt.Sprintf("%08x", num) } + for num := from; num <= to; num++ { + out = append(out, bstream.TestBlockWithLIBNum(id(num), id(num-1), num-100)) + } + return out +} + +func TestForkableHub_LiveLinkingSearchesMergedBlocksOnce(t *testing.T) { + fh, mergedBlocksStore := newMergedBlocksTestHub(t, nil, []uint64{100, 200, 300}, 2000) + store := observeStore(mergedBlocksStore) + + processBlocks(t, fh, liveChainWithoutLIB(901, 901)) + require.False(t, fh.forkable.ForkDBHasLib()) + searchCalls := len(store.seen()) + require.NotZero(t, searchCalls) + + processBlocks(t, fh, liveChainWithoutLIB(902, 950)) + require.False(t, fh.forkable.ForkDBHasLib()) + assert.Len(t, store.seen(), searchCalls, "later blocks reuse what the first search found") + assert.Equal(t, uint64(300), func() uint64 { start, _ := fh.mergedBlocks.knownStart(850); return start }()) +} + +func TestForkableHub_LiveLinkingWaitsAfterFailedSearch(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + t.Cleanup(func() { bootstrapRetryDelay = prev }) + + fh, mergedBlocksStore := newMergedBlocksTestHub(t, nil, []uint64{100, 200, 300}, 2000) + store := observeStore(mergedBlocksStore) + store.setFailing(true) + + processBlocks(t, fh, liveChainWithoutLIB(901, 950)) + assert.Len(t, store.seen(), bootstrapMaxAttempts, "one failed search, then none until mergedBlocksPollInterval passed") + + fh.mergedBlocks.lookupFailedAt.Store(time.Now().Add(-mergedBlocksPollInterval).UnixNano()) + processBlocks(t, fh, liveChainWithoutLIB(951, 951)) + assert.Len(t, store.seen(), 2*bootstrapMaxAttempts, "searches again once mergedBlocksPollInterval passed") +} + +func TestMergedBlocksTracker_KnownStart(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + tracker := &mergedBlocksTracker{bundleSize: 100, maxKeptFinalBlocks: 2000} + _, ok := tracker.knownStart(800) + assert.False(t, ok, "not known before a search") + + tracker.known.Store(true) + tracker.nextBase.Store(0) + start, _ := tracker.knownStart(800) + assert.Equal(t, uint64(1), start, "no merged-blocks file: first streamable block") + + tracker.nextBase.Store(400) + start, _ = tracker.knownStart(800) + assert.Equal(t, uint64(300), start, "start of the last merged-blocks file") + + start, _ = tracker.knownStart(5000) + assert.Equal(t, uint64(3000), start, "never more than maxKeptFinalBlocks below LIB") +} + +func TestMergedBlocksTracker_FollowsMergerFromFarBehind(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 0 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + // the store does not start at the first streamable block, and the merger is far behind + store := dstore.NewMockStore(nil) + for base := uint64(10000); base <= 10400; base += 100 { + setMergedBlocksFiles(store, base) + } + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} + + lowest, err := tracker.locate(context.Background(), 20010) + require.NoError(t, err) + assert.Equal(t, uint64(18100), lowest) + assert.Equal(t, uint64(18100), tracker.nextBase.Load(), "looks no further than maxKeptFinalBlocks below LIB") + + // the merger catches up, writing its files in order + for base := uint64(10500); base <= 19900; base += 100 { + setMergedBlocksFiles(store, base) + } + require.NoError(t, tracker.advance(context.Background(), 20010)) + assert.Equal(t, uint64(20000), tracker.nextBase.Load()) + assert.Equal(t, uint64(11), tracker.lag(20010)) +} + +func TestForkableHub_TrackerLocatesOnceStoreAnswers(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Millisecond + t.Cleanup(func() { bootstrapRetryDelay = prev }) + prevPoll := mergedBlocksPollInterval + mergedBlocksPollInterval = 10 * time.Millisecond + t.Cleanup(func() { mergedBlocksPollInterval = prevPoll }) + + fh, mergedBlocksStore := newMergedBlocksTestHub(t, testChain(100, 900), []uint64{100, 200, 300}, 2000) + store := observeStore(mergedBlocksStore) + store.setFailing(true) + + require.NoError(t, fh.bootstrap()) + _, ok := fh.MergedBlocksUpTo() + require.False(t, ok) + + close(fh.Ready) + store.setFailing(false) + runTracker(t, fh) + + require.Eventually(t, func() bool { _, ok := fh.MergedBlocksUpTo(); return ok }, time.Second, 5*time.Millisecond) + upTo, _ := fh.MergedBlocksUpTo() + assert.Equal(t, uint64(400), upTo) +} + +// runTracker runs the hub's merged-blocks tracker until the test ends, and waits for it +// to stop before the test restores the globals it reads. +func runTracker(t *testing.T, fh *ForkableHub) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + fh.trackMergedBlocks() + }() + t.Cleanup(func() { + fh.Shutdown(nil) + <-done + }) +} + +func newObservedMergedBlocksTestHub(t *testing.T, oneBlocks []*pbbstream.Block, mergedBases []uint64, extra ...Option) (*ForkableHub, *dstore.MockStore, *dstore.MockStore, *observer.ObservedLogs) { + t.Helper() + + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 1 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + oneBlocksStore := dstore.NewMockStore(nil) + AddToMockStore(t, oneBlocksStore, oneBlocks...) + mergedBlocksStore := dstore.NewMockStore(nil) + setMergedBlocksFiles(mergedBlocksStore, mergedBases...) + + core, logs := observer.New(zapcore.WarnLevel) + options := append([]Option{WithMergedBlocksStore(mergedBlocksStore, 100, 2000), WithLogger(zap.New(core))}, extra...) + fh := NewForkableHubWithOptions(bstream.NewTestSourceFactory().NewSource, 0, oneBlocksStore, options) + return fh, oneBlocksStore, mergedBlocksStore, logs +} + +func TestForkableHub_LogsMissingOneBlocksWhileNotReady(t *testing.T) { + prev := notLinkingLogInterval + notLinkingLogInterval = 0 + t.Cleanup(func() { notLinkingLogInterval = prev }) + + // one-block files 400 to 449 are missing, so the merger stopped after file 300 + fh, oneBlocksStore, _, logs := newObservedMergedBlocksTestHub(t, append(testChain(300, 399), testChain(450, 900)...), []uint64{100, 200, 300}) + + require.Error(t, fh.bootstrap()) + processBlocks(t, fh, testChain(901, 905)) + require.False(t, fh.IsReady(), "does not skip over the missing blocks") + + notLinking := logs.FilterMessageSnippet("one-block files are missing").All() + require.NotEmpty(t, notLinking) + fields := notLinking[0].ContextMap() + assert.Equal(t, uint64(400), fields["missing_from_block"]) + assert.Equal(t, uint64(449), fields["missing_up_to_block"]) + + AddToMockStore(t, oneBlocksStore, testChain(400, 449)...) + processBlocks(t, fh, testChain(906, 906)) + assert.True(t, fh.IsReady(), "gets ready once the missing files show up") + assert.Equal(t, uint64(300), fh.forkable.LowestBlockNum()) +} + +func TestForkableHub_NoMissingOneBlocksLogWhenLinking(t *testing.T) { + prev := notLinkingLogInterval + notLinkingLogInterval = 0 + t.Cleanup(func() { notLinkingLogInterval = prev }) + + fh, _, _, logs := newObservedMergedBlocksTestHub(t, testChain(300, 900), []uint64{100, 200, 300}) + require.NoError(t, fh.bootstrap()) + close(fh.Ready) + processBlocks(t, fh, testChain(901, 905)) + assert.Empty(t, logs.FilterMessageSnippet("one-block files are missing").All()) +} + +type testUint64Metric struct{ value atomic.Uint64 } + +func (m *testUint64Metric) SetUint64(value uint64) { m.value.Store(value) } + +func TestForkableHub_TrackerSetsLagAndWarnsWhenBehind(t *testing.T) { + prev := mergedBlocksPollInterval + mergedBlocksPollInterval = 5 * time.Millisecond + t.Cleanup(func() { mergedBlocksPollInterval = prev }) + + metric := &testUint64Metric{} + fh, _, _, logs := newObservedMergedBlocksTestHub(t, testChain(300, 900), []uint64{100, 200, 300}, WithMergedBlocksLagMetric(metric)) + require.NoError(t, fh.bootstrap()) + close(fh.Ready) + runTracker(t, fh) + + // LIB is 898 and the merged blocks end at 399 + require.Eventually(t, func() bool { return metric.value.Load() == 499 }, time.Second, 5*time.Millisecond) + require.Eventually(t, func() bool { return logs.FilterMessageSnippet("merged blocks are behind").Len() > 0 }, time.Second, 5*time.Millisecond) +} + +func TestForkableHub_MergedBlocksLookupTimesOut(t *testing.T) { + prevTimeout := mergedBlocksLookupTimeout + mergedBlocksLookupTimeout = 20 * time.Millisecond + t.Cleanup(func() { mergedBlocksLookupTimeout = prevTimeout }) + prevPoll := mergedBlocksPollInterval + mergedBlocksPollInterval = 5 * time.Millisecond + t.Cleanup(func() { mergedBlocksPollInterval = prevPoll }) + + fh, _, mergedBlocksStore, logs := newObservedMergedBlocksTestHub(t, testChain(300, 900), []uint64{100, 200, 300}) + mergedBlocksStore.WalkFromFunc = func(ctx context.Context, _, _ string, _ func(string) error) error { + <-ctx.Done() // a store call that hangs until its context ends + return ctx.Err() + } + + start := time.Now() + require.NoError(t, fh.bootstrap()) + assert.Less(t, time.Since(start), time.Second, "the boot search gives up after mergedBlocksLookupTimeout") + assert.Equal(t, uint64(600), fh.forkable.LowestBlockNum(), "two merged-blocks files below LIB") + + close(fh.Ready) + runTracker(t, fh) + require.Eventually(t, func() bool { + return logs.FilterMessageSnippet("cannot find the last merged-blocks file").Len() >= 2 + }, time.Second, 5*time.Millisecond, "the tracker keeps retrying") +} + +func TestMergedBlocksTracker_FileExistsStopsWaitingOnCancel(t *testing.T) { + prev := bootstrapRetryDelay + bootstrapRetryDelay = time.Hour + t.Cleanup(func() { bootstrapRetryDelay = prev }) + + store := dstore.NewMockStore(nil) + store.FileExistsFunc = func(context.Context, string) (bool, error) { + return false, errors.New("store unavailable") + } + tracker := &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000} + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + _, err := tracker.fileExists(ctx, 100) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(start), time.Second) +} + +func TestMergedBlocksTracker_ListsConsecutiveFiles(t *testing.T) { + prev := bstream.GetProtocolFirstStreamableBlock + bstream.GetProtocolFirstStreamableBlock = 0 + t.Cleanup(func() { bstream.GetProtocolFirstStreamableBlock = prev }) + + newTracker := func(bases ...uint64) (*mergedBlocksTracker, *dstore.MockStore) { + store := dstore.NewMockStore(nil) + setMergedBlocksFiles(store, bases...) + return &mergedBlocksTracker{store: store, bundleSize: 100, maxKeptFinalBlocks: 2000}, store + } + + t.Run("a file written ahead of a missing one is not the end of merged blocks", func(t *testing.T) { + tracker, _ := newTracker(300, 400, 500, 700) + start, err := tracker.locate(context.Background(), 950) + require.NoError(t, err) + assert.Equal(t, uint64(500), start) + assert.Equal(t, uint64(600), tracker.nextBase.Load()) + }) + + t.Run("a store that does not start at the first streamable block", func(t *testing.T) { + tracker, _ := newTracker(600, 700) + start, err := tracker.locate(context.Background(), 950) + require.NoError(t, err) + assert.Equal(t, uint64(700), start) + assert.Equal(t, uint64(800), tracker.nextBase.Load()) + }) + + t.Run("other files of the store are ignored", func(t *testing.T) { + tracker, store := newTracker(300, 400) + store.SetFile("0000000450", []byte("not on a file boundary")) + store.SetFile("0000000500.tmp", []byte("merged")) + store.SetFile("index", []byte("not a number")) + start, err := tracker.locate(context.Background(), 950) + require.NoError(t, err) + assert.Equal(t, uint64(500), start) + assert.Equal(t, uint64(600), tracker.nextBase.Load()) + }) + + t.Run("following a merger that catches up stops at the first missing file", func(t *testing.T) { + tracker, store := newTracker(300) + _, err := tracker.locate(context.Background(), 2000) + require.NoError(t, err) + require.Equal(t, uint64(400), tracker.nextBase.Load()) + + setMergedBlocksFiles(store, 500, 600) + require.NoError(t, tracker.advance(context.Background(), 2000)) + assert.Equal(t, uint64(400), tracker.nextBase.Load(), "the next file is not there") + + setMergedBlocksFiles(store, 400, 800) + require.NoError(t, tracker.advance(context.Background(), 2000)) + assert.Equal(t, uint64(700), tracker.nextBase.Load()) + }) +} diff --git a/hub/subscription_test.go b/hub/subscription_test.go index 3055d97..aa5ff94 100644 --- a/hub/subscription_test.go +++ b/hub/subscription_test.go @@ -194,6 +194,13 @@ func TestSubscription_Run_WithPartialBlocks(t *testing.T) { handler := &testHandler{} sub := NewSubscription(handler, 10, true) + // Fill the channel before run() starts: which partial blocks it skips depends + // on what is already waiting behind them. + for _, block := range tt.blocks { + err := sub.push(block) + require.NoError(t, err) + } + // Start run() in a goroutine var runErr error var wg sync.WaitGroup @@ -203,12 +210,6 @@ func TestSubscription_Run_WithPartialBlocks(t *testing.T) { runErr = sub.run() }() - // Fill the channel with test blocks - for _, block := range tt.blocks { - err := sub.push(block) - require.NoError(t, err) - } - // Give some time for blocks to be processed time.Sleep(10 * time.Millisecond)