diff --git a/CHANGELOG.md b/CHANGELOG.md index d9810ea..1ad235f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `bstream_blockstream_subscriptions_closed_full_total` (`blockstream.SubscriptionsClosedFull`): counts block stream subscriptions closed because the subscriber (e.g. a relayer connected to a reader) fell more blocks behind than the subscription channel size. Each one used to show only as a `reach max buffer size for subcription` log line. Exported by processes that create a block stream server. - Add `hub.WithMergedBlocksStore(store, bundleSize, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`. - A hub that is not ready because one-block files are missing between the blocks it loaded and the live blocks logs the missing range every 30s. It waits for those files instead of starting above them, so streams going from merged blocks to live blocks never skip blocks. - Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for. diff --git a/blockstream/metrics.go b/blockstream/metrics.go new file mode 100644 index 0000000..6731f09 --- /dev/null +++ b/blockstream/metrics.go @@ -0,0 +1,28 @@ +// Copyright 2019 dfuse Platform Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blockstream + +import ( + "github.com/streamingfast/dmetrics" +) + +// metrics is registered when a block stream server is created, so a process exports these +// metrics only when it serves a block stream (e.g. a reader serving its relayers). +var metrics = dmetrics.NewSet(dmetrics.PrefixNameWith("bstream_blockstream")) + +// SubscriptionsClosedFull counts subscriptions closed because their channel was full: the +// subscriber (e.g. a relayer) fell more than the subscription channel size behind and was +// disconnected. See ServerOptionWithSubscriptionChannelSize. +var SubscriptionsClosedFull = metrics.NewCounter("subscriptions_closed_full_total", "Number of block stream subscriptions closed because the subscriber fell more blocks behind than the subscription channel size") diff --git a/blockstream/server.go b/blockstream/server.go index 0c8eb62..ecdbf59 100644 --- a/blockstream/server.go +++ b/blockstream/server.go @@ -98,6 +98,8 @@ func NewServer(server dgrpcserver.Server, opts ...ServerOption) *Server { // // Moreover, management methods like `Close` func NewUnmanagedServer(opts ...ServerOption) *Server { + metrics.Register() + s := &Server{ logger: zlog, subscriptionChanSize: DefaultSubscriptionChannelSize, diff --git a/blockstream/server_test.go b/blockstream/server_test.go index 6eadfb3..fcea9bc 100644 --- a/blockstream/server_test.go +++ b/blockstream/server_test.go @@ -18,6 +18,8 @@ import ( "fmt" "testing" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/streamingfast/bstream" dgrpcserver "github.com/streamingfast/dgrpc/server" dgrpcstandard "github.com/streamingfast/dgrpc/server/standard" @@ -58,12 +60,33 @@ func TestBlockServerSubscriptionChannelSize(t *testing.T) { func TestBlockServerSubscriptionClosedWhenFull(t *testing.T) { s := NewUnmanagedServer(ServerOptionWithSubscriptionChannelSize(3)) sub := s.subscribe(0, "slow") + closedBefore := testutil.ToFloat64(SubscriptionsClosedFull) for i := 0; i < 3; i++ { require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers(fmt.Sprintf("%08xa", i+2), fmt.Sprintf("%08xa", i+1), uint64(i+2), 0))) } require.False(t, sub.closed, "subscription should hold up to its capacity") + require.Equal(t, closedBefore, testutil.ToFloat64(SubscriptionsClosedFull)) require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers("00000005a", "00000004a", 5, 0))) require.True(t, sub.closed, "subscription should close once over capacity") + require.Equal(t, closedBefore+1, testutil.ToFloat64(SubscriptionsClosedFull), "closing a full subscription is counted once") + + // Blocks pushed after the close are not counted again. + require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers("00000006a", "00000005a", 6, 0))) + require.Equal(t, closedBefore+1, testutil.ToFloat64(SubscriptionsClosedFull)) +} + +// Creating a server registers the metrics, so a process serving a block stream exports them. +func TestBlockServerRegistersMetrics(t *testing.T) { + NewUnmanagedServer() + + families, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + + exported := map[string]bool{} + for _, family := range families { + exported[family.GetName()] = true + } + require.True(t, exported["bstream_blockstream_subscriptions_closed_full_total"]) } diff --git a/blockstream/subscription.go b/blockstream/subscription.go index 94f8bf3..efe0bd8 100644 --- a/blockstream/subscription.go +++ b/blockstream/subscription.go @@ -45,6 +45,7 @@ func (s *subscription) SetLogger(logger *zap.Logger) { func (s *subscription) Push(blk *pbbstream.Block) { if len(s.incomingBlock) == cap(s.incomingBlock) { s.quitOnce.Do(func() { + SubscriptionsClosedFull.Inc() s.logger.Info("reach max buffer size for subcription, closing channel", zap.Int("capacity", cap(s.incomingBlock))) s.closed = true close(s.incomingBlock) diff --git a/go.mod b/go.mod index ffd73b2..fe50707 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/mostynb/go-grpc-compression v1.2.3 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 + github.com/prometheus/client_golang v1.23.2 github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993 github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62 @@ -100,7 +101,6 @@ require ( github.com/paulbellamy/ratecounter v0.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect