Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

- Add `hub.WithMergedBlocksStore(store, bundleSize, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`.
- A hub that is not ready because one-block files are missing between the blocks it loaded and the live blocks logs the missing range every 30s. It waits for those files instead of starting above them, so streams going from merged blocks to live blocks never skip blocks.
- Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for.
- `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once.
- `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once.
- `ForkableHub.WithoutPartials` and `stream.WithoutPartialBlocks`: stream from a hub fed with partial blocks without getting them, the last partial of each block arriving as the complete block, as for a relayer client that does not ask for partials. Lets an app that does not handle partial blocks share its hub with one that does.
Expand Down
25 changes: 24 additions & 1 deletion forkable/forkable.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ type Forkable struct {
holdBlocksUntilLIB bool // if true, never passthrough anything before a LIB is set
keptFinalBlocks int // how many blocks we keep behind LIB

// finalBlocksPurgeLimit, when set, can lower the block number under which final
// blocks are dropped, see WithFinalBlocksPurgeLimit.
finalBlocksPurgeLimit func(libNum, cutoff uint64) uint64

includeInitialLIB bool

failOnUnlinkableBlocksCount int
Expand Down Expand Up @@ -860,7 +864,7 @@ func (p *Forkable) ProcessBlock(blk *pbbstream.Block, obj any) error {
p.forkDB.MoveLIB(libRef)
headBlockNum := newHeadBlock.Num()

_ = p.forkDB.PurgeBeforeLIBAndPartials(p.keptFinalBlocks, &headBlockNum)
_ = p.forkDB.PurgeBeforeLIBAndPartials(p.finalBlocksToKeep(libRef.Num()), &headBlockNum)

if err := p.processIrreversibleSegment(irreversibleSegment, ppBlk.Block.AsRef()); err != nil {
return err
Expand All @@ -873,6 +877,25 @@ func (p *Forkable) ProcessBlock(blk *pbbstream.Block, obj any) error {
return nil
}

// finalBlocksToKeep is how many blocks are kept below a LIB at libNum: keptFinalBlocks,
// or more when finalBlocksPurgeLimit holds the purge back.
func (p *Forkable) finalBlocksToKeep(libNum uint64) int {
if p.finalBlocksPurgeLimit == nil {
return p.keptFinalBlocks
}

var cutoff uint64
if libNum > uint64(p.keptFinalBlocks) {
cutoff = libNum - uint64(p.keptFinalBlocks)
}

limit := p.finalBlocksPurgeLimit(libNum, cutoff)
if limit >= cutoff {
return p.keptFinalBlocks
}
return int(libNum - limit)
}

func ids(blocks []*ForkableBlock) (ids []string) {
ids = make([]string, len(blocks))
for i, obj := range blocks {
Expand Down
12 changes: 12 additions & 0 deletions forkable/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,18 @@ func WithKeptFinalBlocks(count int) Option {
}
}

// WithFinalBlocksPurgeLimit lets the caller keep more final blocks than WithKeptFinalBlocks
// asks for. Each time the LIB moves, limit is called with the new LIB number and the
// cutoff the forkable is about to apply (blocks below it are dropped); returning a lower
// number keeps the blocks from that number on. A higher number is ignored.
//
// limit is called with the forkable locked, on the path of every block: it must not block.
func WithFinalBlocksPurgeLimit(limit func(libNum, cutoff uint64) uint64) Option {
return func(f *Forkable) {
f.finalBlocksPurgeLimit = limit
}
}

func EnsureBlockFlows(blockRef bstream.BlockRef) Option {
return func(f *Forkable) {
f.ensureBlockFlows = blockRef
Expand Down
104 changes: 55 additions & 49 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/streamingfast/bstream

go 1.25.0
go 1.26.0

require (
github.com/RoaringBitmap/roaring v1.9.4
Expand All @@ -10,7 +10,7 @@ require (
github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c
github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993
github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62
github.com/streamingfast/dstore v0.2.3
github.com/streamingfast/dstore v0.2.4-0.20261113015734-ed2936b371f8
github.com/streamingfast/logging v1.2.2
github.com/streamingfast/opaque v0.0.0-20210811180740-0c01d37ea308
github.com/streamingfast/pbgo v0.0.6-0.20251125204657-0a9c67563b19
Expand All @@ -28,42 +28,42 @@ require (
cloud.google.com/go/auth v0.20.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.7.0 // indirect
cloud.google.com/go/logging v1.17.0 // indirect
cloud.google.com/go/longrunning v0.9.0 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
cloud.google.com/go/storage v1.62.0 // indirect
cloud.google.com/go/trace v1.11.7 // indirect
cloud.google.com/go/iam v1.12.0 // indirect
cloud.google.com/go/monitoring v1.30.0 // indirect
cloud.google.com/go/storage v1.69.0 // indirect
cloud.google.com/go/trace v1.16.0 // indirect
connectrpc.com/connect v1.19.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
github.com/aws/aws-sdk-go-v2/config v1.29.12 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.65 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect
github.com/aws/smithy-go v1.24.2 // indirect
github.com/apache/arrow-go/v18 v18.7.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
github.com/aws/aws-sdk-go-v2/config v1.33.6 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect
github.com/aws/smithy-go v1.28.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.12.0 // indirect
github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d // indirect
Expand All @@ -78,23 +78,27 @@ require (
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
github.com/goccy/go-json v0.10.6 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/google/flatbuffers v25.12.19+incompatible // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect
github.com/googleapis/gax-go/v2 v2.26.2 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/klauspost/compress v1.20.1 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
github.com/paulbellamy/ratecounter v0.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.28 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
Expand All @@ -104,9 +108,10 @@ require (
github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
github.com/streamingfast/sf-tracing v0.0.0-20251218140752-bafd5572499f // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
go.opentelemetry.io/otel v1.45.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
Expand All @@ -122,16 +127,17 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect
golang.org/x/net v0.59.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.40.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/term v0.46.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/api v0.278.0 // indirect
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
google.golang.org/api v0.288.0 // indirect
google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
)
Loading
Loading