Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

- `bstream_blockstream_subscriptions_closed_full_total` (`blockstream.SubscriptionsClosedFull`): counts block stream subscriptions closed because the subscriber (e.g. a relayer connected to a reader) fell more blocks behind than the subscription channel size. Each one used to show only as a `reach max buffer size for subcription` log line. Exported by processes that create a block stream server.
- Add `hub.WithMergedBlocksStore(store, bundleSize, maxKeptFinalBlocks)`: the hub keeps every final block from the start of the last merged-blocks file, instead of a fixed `keepFinalBlocks` below LIB (pass 0), so a merger that is behind no longer leaves blocks that neither the files nor the hub can serve. Related: `hub.WithMergedBlocksLagMetric`, `ForkableHub.MergedBlocksUpTo`.
- A hub that is not ready because one-block files are missing between the blocks it loaded and the live blocks logs the missing range every 30s. It waits for those files instead of starting above them, so streams going from merged blocks to live blocks never skip blocks.
- Add `forkable.WithFinalBlocksPurgeLimit` to keep more final blocks than `forkable.WithKeptFinalBlocks` asks for.
Expand Down
28 changes: 28 additions & 0 deletions blockstream/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright 2019 dfuse Platform Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blockstream

import (
"github.com/streamingfast/dmetrics"
)

// metrics is registered when a block stream server is created, so a process exports these
// metrics only when it serves a block stream (e.g. a reader serving its relayers).
var metrics = dmetrics.NewSet(dmetrics.PrefixNameWith("bstream_blockstream"))

// SubscriptionsClosedFull counts subscriptions closed because their channel was full: the
// subscriber (e.g. a relayer) fell more than the subscription channel size behind and was
// disconnected. See ServerOptionWithSubscriptionChannelSize.
var SubscriptionsClosedFull = metrics.NewCounter("subscriptions_closed_full_total", "Number of block stream subscriptions closed because the subscriber fell more blocks behind than the subscription channel size")
2 changes: 2 additions & 0 deletions blockstream/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ func NewServer(server dgrpcserver.Server, opts ...ServerOption) *Server {
//
// Moreover, management methods like `Close`
func NewUnmanagedServer(opts ...ServerOption) *Server {
metrics.Register()

s := &Server{
logger: zlog,
subscriptionChanSize: DefaultSubscriptionChannelSize,
Expand Down
23 changes: 23 additions & 0 deletions blockstream/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"fmt"
"testing"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/streamingfast/bstream"
dgrpcserver "github.com/streamingfast/dgrpc/server"
dgrpcstandard "github.com/streamingfast/dgrpc/server/standard"
Expand Down Expand Up @@ -58,12 +60,33 @@ func TestBlockServerSubscriptionChannelSize(t *testing.T) {
func TestBlockServerSubscriptionClosedWhenFull(t *testing.T) {
s := NewUnmanagedServer(ServerOptionWithSubscriptionChannelSize(3))
sub := s.subscribe(0, "slow")
closedBefore := testutil.ToFloat64(SubscriptionsClosedFull)

for i := 0; i < 3; i++ {
require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers(fmt.Sprintf("%08xa", i+2), fmt.Sprintf("%08xa", i+1), uint64(i+2), 0)))
}
require.False(t, sub.closed, "subscription should hold up to its capacity")
require.Equal(t, closedBefore, testutil.ToFloat64(SubscriptionsClosedFull))

require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers("00000005a", "00000004a", 5, 0)))
require.True(t, sub.closed, "subscription should close once over capacity")
require.Equal(t, closedBefore+1, testutil.ToFloat64(SubscriptionsClosedFull), "closing a full subscription is counted once")

// Blocks pushed after the close are not counted again.
require.NoError(t, s.PushBlock(bstream.TestBlockWithNumbers("00000006a", "00000005a", 6, 0)))
require.Equal(t, closedBefore+1, testutil.ToFloat64(SubscriptionsClosedFull))
}

// Creating a server registers the metrics, so a process serving a block stream exports them.
func TestBlockServerRegistersMetrics(t *testing.T) {
NewUnmanagedServer()

families, err := prometheus.DefaultGatherer.Gather()
require.NoError(t, err)

exported := map[string]bool{}
for _, family := range families {
exported[family.GetName()] = true
}
require.True(t, exported["bstream_blockstream_subscriptions_closed_full_total"])
}
1 change: 1 addition & 0 deletions blockstream/subscription.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ func (s *subscription) SetLogger(logger *zap.Logger) {
func (s *subscription) Push(blk *pbbstream.Block) {
if len(s.incomingBlock) == cap(s.incomingBlock) {
s.quitOnce.Do(func() {
SubscriptionsClosedFull.Inc()
s.logger.Info("reach max buffer size for subcription, closing channel", zap.Int("capacity", cap(s.incomingBlock)))
s.closed = true
close(s.incomingBlock)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/golang/protobuf v1.5.4
github.com/mostynb/go-grpc-compression v1.2.3
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10
github.com/prometheus/client_golang v1.23.2
github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c
github.com/streamingfast/dgrpc v0.0.0-20260420180129-8b81f2664993
github.com/streamingfast/dmetrics v0.0.0-20260109212625-35256f512c62
Expand Down Expand Up @@ -100,7 +101,6 @@ require (
github.com/paulbellamy/ratecounter v0.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.28 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
Expand Down
Loading