From 9ed5e3d04c0c974b3520bbe5171a7c997b24b8f6 Mon Sep 17 00:00:00 2001 From: Matthew Date: Thu, 12 Nov 2026 11:55:47 -0500 Subject: [PATCH] blockstream: count subscriptions closed because they were full A subscriber that falls more blocks behind than the subscription channel size is disconnected, which so far only showed as a log line. Add bstream_blockstream_subscriptions_closed_full_total, incremented once per closed subscription and registered when a block stream server is created. Also remove a duplicated CHANGELOG entry for ServerOptionWithSubscriptionChannelSize. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- blockstream/metrics.go | 28 ++++++++++++++++++++++++++++ blockstream/server.go | 2 ++ blockstream/server_test.go | 23 +++++++++++++++++++++++ blockstream/subscription.go | 1 + go.mod | 2 +- 6 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 blockstream/metrics.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 588dcad..b55328d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. +- `bstream_blockstream_subscriptions_closed_full_total` (`blockstream.SubscriptionsClosedFull`): counts block stream subscriptions closed because the subscriber (e.g. a relayer connected to a reader) fell more blocks behind than the subscription channel size. Each one used to show only as a `reach max buffer size for subcription` log line. Exported by processes that create a block stream server. - `blockstream.ServerOptionWithSubscriptionChannelSize` (default `blockstream.DefaultSubscriptionChannelSize`, `200`): how many blocks a block stream subscriber (e.g. a relayer connected to a reader) can lag behind before its subscription is closed. It was hard-coded to `200`, which a reader catching up after a stall can exceed in one burst, dropping every relayer at once. - `ForkableHub.WithoutPartials` and `stream.WithoutPartialBlocks`: stream from a hub fed with partial blocks without getting them, the last partial of each block arriving as the complete block, as for a relayer client that does not ask for partials. Lets an app that does not handle partial blocks share its hub with one that does. - `hub.SubscriptionCatchUpTimeout` (default `30s`): a hub subscription is now closed with `hub.ErrSubscriptionBehind` when its consumer has not emptied its waiting blocks for that long, which catches a consumer that is stuck or slower than the chain. A consumer draining a burst within the timeout is not affected. `ErrSubscriptionBehind` matches `ErrSubscriptionChannelFull` with `errors.Is`, so callers that reconnect on one reconnect on both. diff --git a/blockstream/metrics.go b/blockstream/metrics.go new file mode 100644 index 0000000..6731f09 --- /dev/null +++ b/blockstream/metrics.go @@ -0,0 +1,28 @@ +// Copyright 2019 dfuse Platform Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blockstream + +import ( + "github.com/streamingfast/dmetrics" +) + +// metrics is registered when a block stream server is created, so a process exports these +// metrics only when it serves a block stream (e.g. a reader serving its relayers). +var metrics = dmetrics.NewSet(dmetrics.PrefixNameWith("bstream_blockstream")) + +// SubscriptionsClosedFull counts subscriptions closed because their channel was full: the +// subscriber (e.g. a relayer) fell more than the subscription channel size behind and was +// disconnected. See ServerOptionWithSubscriptionChannelSize. +var SubscriptionsClosedFull = metrics.NewCounter("subscriptions_closed_full_total", "Number of block stream subscriptions closed because the subscriber fell more blocks behind than the subscription channel size") diff --git a/blockstream/server.go b/blockstream/server.go index 0c8eb62..ecdbf59 100644 --- a/blockstream/server.go +++ b/blockstream/server.go @@ -98,6 +98,8 @@ func NewServer(server dgrpcserver.Server, opts ...ServerOption) *Server { // // Moreover, management methods like `Close` func NewUnmanagedServer(opts ...ServerOption) *Server { + metrics.Register() + s := &Server{ logger: zlog, subscriptionChanSize: DefaultSubscriptionChannelSize, diff --git a/blockstream/server_test.go b/blockstream/server_test.go index 6eadfb3..fcea9bc 100644 --- a/blockstream/server_test.go +++ b/blockstream/server_test.go @@ -18,6 +18,8 @@ import ( "fmt" "testing" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/streamingfast/bstream" dgrpcserver "github.com/streamingfast/dgrpc/server" dgrpcstandard "github.com/streamingfast/dgrpc/server/standard" @@ -58,12 +60,33 @@ func TestBlockServerSubscriptionChannelSize(t *testing.T) { func TestBlockServerSubscriptionClosedWhenFull(t *testing.T) { s := NewUnmanagedServer(ServerOptionWithSubscriptionChannelSize(3)) sub := s.subscribe(0, "slow") + closedBefore := testutil.ToFloat64(SubscriptionsClosedFull) for i := 0; i < 3; i++ { require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers(fmt.Sprintf("%08xa", i+2), fmt.Sprintf("%08xa", i+1), uint64(i+2), 0))) } require.False(t, sub.closed, "subscription should hold up to its capacity") + require.Equal(t, closedBefore, testutil.ToFloat64(SubscriptionsClosedFull)) require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers("00000005a", "00000004a", 5, 0))) require.True(t, sub.closed, "subscription should close once over capacity") + require.Equal(t, closedBefore+1, testutil.ToFloat64(SubscriptionsClosedFull), "closing a full subscription is counted once") + + // Blocks pushed after the close are not counted again. + require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers("00000006a", "00000005a", 6, 0))) + require.Equal(t, closedBefore+1, testutil.ToFloat64(SubscriptionsClosedFull)) +} + +// Creating a server registers the metrics, so a process serving a block stream exports them. +func TestBlockServerRegistersMetrics(t *testing.T) { + NewUnmanagedServer() + + families, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + + exported := map[string]bool{} + for _, family := range families { + exported[family.GetName()] = true + } + require.True(t, exported["bstream_blockstream_subscriptions_closed_full_total"]) } diff --git a/blockstream/subscription.go b/blockstream/subscription.go index 94f8bf3..efe0bd8 100644 --- a/blockstream/subscription.go +++ b/blockstream/subscription.go @@ -45,6 +45,7 @@ func (s *subscription) SetLogger(logger *zap.Logger) { func (s *subscription) Push(blk *pbbstream.Block) { if len(s.incomingBlock) == cap(s.incomingBlock) { s.quitOnce.Do(func() { + SubscriptionsClosedFull.Inc() s.logger.Info("reach max buffer size for subcription, closing channel", zap.Int("capacity", cap(s.incomingBlock))) s.closed = true close(s.incomingBlock) diff --git a/go.mod b/go.mod index 0cccd95..0bb15ea 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/mostynb/go-grpc-compression v1.2.3 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 + github.com/prometheus/client_golang v1.23.2 github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993 github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62 @@ -96,7 +97,6 @@ require ( github.com/openzipkin/zipkin-go v0.4.3 // indirect github.com/paulbellamy/ratecounter v0.2.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect