From 24eb12b3b33789f4d1cb222257a7994acc3bb23a Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:59:24 -0600 Subject: [PATCH 01/29] feat(bps): initial commit --- pkg/bps/bps.go | 161 +++ pkg/bps/bps_test.go | 85 ++ pkg/bps/main_test.go | 15 + pkg/bps/pb/bps.pb.go | 2349 ++++++++++++++++++++++++++++++++++++++ pkg/bps/pb/bps.proto | 103 ++ pkg/bps/pb/doc.go | 8 + pkg/bps/registry.go | 33 + pkg/bps/registry_test.go | 1 + 8 files changed, 2755 insertions(+) create mode 100644 pkg/bps/bps.go create mode 100644 pkg/bps/bps_test.go create mode 100644 pkg/bps/main_test.go create mode 100644 pkg/bps/pb/bps.pb.go create mode 100644 pkg/bps/pb/bps.proto create mode 100644 pkg/bps/pb/doc.go create mode 100644 pkg/bps/registry.go create mode 100644 pkg/bps/registry_test.go diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go new file mode 100644 index 00000000000..9124020c9db --- /dev/null +++ b/pkg/bps/bps.go @@ -0,0 +1,161 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bps exposes a simple hello-world wire protocol +// used to greet other peers. +package bps + +import ( + "context" + "fmt" + "sync" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// loggerName is the tree path name of the logger for this package. +const loggerName = "bps" + +const ( + protocolName = "bps" + protocolVersion = "1.0.0" + streamNameSystem = "bps-system" + streamNameMessage = "bps-message" +) + +// Interface is the main interface of the bps protocol. +type Interface interface { + Join(ctx context.Context, p p2p.Peer, topic []byte, notify <-chan []byte) error + Publish(ctx context.Context, topic []byte, message []byte) error +} + +// Service is the bps protocol service. +type Service struct { + mtx sync.Mutex + streamer p2p.Streamer + // cohort registry - keep track of members, publisher, channels, challenge, streams? + registry *CohortRegistry + + // joined + joined map[string]chan []byte + + logger log.Logger +} + +// New returns a new bps Service. +func New(streamer p2p.Streamer, logger log.Logger) *Service { + return &Service{ + streamer: streamer, + registry: NewRegistry(), + joined: make(map[string]chan []byte), + logger: logger.WithName(loggerName).Register(), + } +} + +// Protocol returns the p2p protocol specification for bps. +func (s *Service) Protocol() p2p.ProtocolSpec { + return p2p.ProtocolSpec{ + Name: protocolName, + Version: protocolVersion, + StreamSpecs: []p2p.StreamSpec{ + { + Name: streamNameSystem, + Handler: s.handlerSystem, + }, { + Name: streamNameMessage, + Handler: s.handlerMessage, + }, + }, + } +} + +// Join onto a topic at the remote peer. The channel passed at registration will be notified +// with the payloads once they arrive. Returns the challenge and a channel that would send +// the payloads over it +func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) ([]byte, chan []byte, error) { + stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamNameSystem) + if err != nil { + return nil, nil, fmt.Errorf("new stream: %w", err) + } + defer func() { + go stream.FullClose() + }() + + w, r := protobuf.NewWriterAndReader(stream) + joinMsg := pb.SystemMessage{SysMessage: &pb.SystemMessage_Join{ + Join: &pb.Join{Topic: topic}, + }} + if err := w.WriteMsgWithContext(ctx, &joinMsg); err != nil { + return nil, nil, fmt.Errorf("write join msg: %w", err) + } + joinAck := pb.JoinAck{} + if err := r.ReadMsgWithContext(ctx, &joinAck); err != nil { + return nil, nil, fmt.Errorf("read join ack: %w", err) + } + rxCh := make(chan []byte) + s.joined[string(topic)] = rxCh + return joinAck.Challenge, rxCh, nil +} + +func (s *Service) handlerSystem(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { + _, r := protobuf.NewWriterAndReader(stream) + defer stream.FullClose() + + var sysMsg pb.SystemMessage + if err := r.ReadMsgWithContext(ctx, &sysMsg); err != nil { + return fmt.Errorf("read sys message: %w", err) + } + + if join := sysMsg.GetJoin(); join != nil { + // peer is trying to join the cohort. accept and return the challenge + + return nil + } + if claim := sysMsg.GetClaim(); claim != nil { + // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off + + return nil + } + + return nil +} + +func (s *Service) Publish(ctx context.Context, topic []byte, message []byte) error { + // grabs an existing publish stream or creates it then pushes the message down the pipe + // the stream stays alive and is reused later on for future messages + return nil +} + +// handlerMessage handles incoming messages. on non-brokers, it tries to find the +// topic in the joined map, then notify the subscribed reader off the channel. +// the stream is reused and kept with a long-running goroutine. +func (s *Service) handlerMessage(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { + _, r := protobuf.NewWriterAndReader(stream) + //defer stream.FullClose() // NO FULL CLOSE HERE + // + var msg pb.PayloadMessage + if err := r.ReadMsgWithContext(ctx, &msg); err != nil { + return fmt.Errorf("read sys message: %w", err) + } + + if publish := msg.GetPublish(); publish != nil { + // we got a publish message - this means we are a broker. + // if there's a cohort - publish the message to the peers. launch a goroutine + // and read off the channel and feed it to the different readers + return nil + } + if bcast := msg.GetBroadcast(); bcast != nil { + // we got a broadcast on this stream, it means we are a subscriber here. + // find the subscriber channel(s), launch polling goroutine and duly notify + // when something comes in on the stream + + return nil + } + + return nil +} diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go new file mode 100644 index 00000000000..5357816299c --- /dev/null +++ b/pkg/bps/bps_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps_test + +import ( + "bytes" + "context" + "testing" + + "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/p2p/streamtest" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +func TestGreet(t *testing.T) { + t.Parallel() + + logger := log.Noop + + server := bps.New(nil, logger) + + recorder := streamtest.New( + streamtest.WithProtocols(server.Protocol()), + ) + + client := bps.New(recorder, logger) + + addr := swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c") + greeting := "world" + + response, err := client.Greet(context.Background(), addr, greeting) + if err != nil { + t.Fatal(err) + } + + if want := "hello, world"; response != want { + t.Fatalf("got response %q, want %q", response, want) + } + + records, err := recorder.Records(addr, "bps", "1.0.0", "bps") + if err != nil { + t.Fatal(err) + } + if l := len(records); l != 1 { + t.Fatalf("got %v records, want %v", l, 1) + } + record := records[0] + + messages, err := protobuf.ReadMessages( + bytes.NewReader(record.In()), + func() protobuf.Message { return new(pb.Hello) }, + ) + if err != nil { + t.Fatal(err) + } + if l := len(messages); l != 1 { + t.Fatalf("got %v messages, want %v", l, 1) + } + if got := messages[0].(*pb.Hello).Greeting; got != greeting { + t.Fatalf("got greeting %q, want %q", got, greeting) + } + + messages, err = protobuf.ReadMessages( + bytes.NewReader(record.Out()), + func() protobuf.Message { return new(pb.Welcome) }, + ) + if err != nil { + t.Fatal(err) + } + if l := len(messages); l != 1 { + t.Fatalf("got %v messages, want %v", l, 1) + } + if got, want := messages[0].(*pb.Welcome).Response, "hello, world"; got != want { + t.Fatalf("got response %q, want %q", got, want) + } + + if err := record.Err(); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/bps/main_test.go b/pkg/bps/main_test.go new file mode 100644 index 00000000000..784030d903a --- /dev/null +++ b/pkg/bps/main_test.go @@ -0,0 +1,15 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps_test + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go new file mode 100644 index 00000000000..c426d596f57 --- /dev/null +++ b/pkg/bps/pb/bps.pb.go @@ -0,0 +1,2349 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: bps.proto + +package pb + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// What the topic binds to. One binding. +type TopicBinding int32 + +const ( + TopicBinding_TOPIC_BINDING_UNSPECIFIED TopicBinding = 0 + TopicBinding_FEED_TOPIC TopicBinding = 4 +) + +var TopicBinding_name = map[int32]string{ + 0: "TOPIC_BINDING_UNSPECIFIED", + 4: "FEED_TOPIC", +} + +var TopicBinding_value = map[string]int32{ + "TOPIC_BINDING_UNSPECIFIED": 0, + "FEED_TOPIC": 4, +} + +func (x TopicBinding) String() string { + return proto.EnumName(TopicBinding_name, int32(x)) +} + +func (TopicBinding) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{0} +} + +type Status int32 + +const ( + Status_STATUS_UNSPECIFIED Status = 0 + Status_OK Status = 1 + Status_FULL Status = 2 + Status_REJECTED Status = 4 +) + +var Status_name = map[int32]string{ + 0: "STATUS_UNSPECIFIED", + 1: "OK", + 2: "FULL", + 4: "REJECTED", +} + +var Status_value = map[string]int32{ + "STATUS_UNSPECIFIED": 0, + "OK": 1, + "FULL": 2, + "REJECTED": 4, +} + +func (x Status) String() string { + return proto.EnumName(Status_name, int32(x)) +} + +func (Status) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{1} +} + +// The cohort's identity. Fixed by whoever joins first; immutable. +type CohortSpec struct { + Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` + Admin []byte `protobuf:"bytes,5,opt,name=admin,proto3" json:"admin,omitempty"` +} + +func (m *CohortSpec) Reset() { *m = CohortSpec{} } +func (m *CohortSpec) String() string { return proto.CompactTextString(m) } +func (*CohortSpec) ProtoMessage() {} +func (*CohortSpec) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{0} +} +func (m *CohortSpec) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CohortSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CohortSpec.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CohortSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_CohortSpec.Merge(m, src) +} +func (m *CohortSpec) XXX_Size() int { + return m.Size() +} +func (m *CohortSpec) XXX_DiscardUnknown() { + xxx_messageInfo_CohortSpec.DiscardUnknown(m) +} + +var xxx_messageInfo_CohortSpec proto.InternalMessageInfo + +func (m *CohortSpec) GetTopic() []byte { + if m != nil { + return m.Topic + } + return nil +} + +func (m *CohortSpec) GetBinding() TopicBinding { + if m != nil { + return m.Binding + } + return TopicBinding_TOPIC_BINDING_UNSPECIFIED +} + +func (m *CohortSpec) GetAdmin() []byte { + if m != nil { + return m.Admin + } + return nil +} + +// Proof of an identity, bound to the stream it arrives on: +// owner = ecrecover( keccak256("bps-join:v1" || topic || admin), signature ) +// The preimage is static and carries no node identity (SWIP-60): the same key works +// from any node, and a join never links an address to a peer id. It is therefore +// replayable, and a replayed identity is worthless: authorship rests on the message +// signature, never on the handshake. +type Auth struct { + Signature []byte `protobuf:"bytes,1,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (m *Auth) Reset() { *m = Auth{} } +func (m *Auth) String() string { return proto.CompactTextString(m) } +func (*Auth) ProtoMessage() {} +func (*Auth) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{1} +} +func (m *Auth) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Auth) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Auth.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Auth) XXX_Merge(src proto.Message) { + xxx_messageInfo_Auth.Merge(m, src) +} +func (m *Auth) XXX_Size() int { + return m.Size() +} +func (m *Auth) XXX_DiscardUnknown() { + xxx_messageInfo_Auth.DiscardUnknown(m) +} + +var xxx_messageInfo_Auth proto.InternalMessageInfo + +func (m *Auth) GetSignature() []byte { + if m != nil { + return m.Signature + } + return nil +} + +// Broker -> peer, answering Join. Status only. A non-OK Ack ends the stream. +type Ack struct { + Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` +} + +func (m *Ack) Reset() { *m = Ack{} } +func (m *Ack) String() string { return proto.CompactTextString(m) } +func (*Ack) ProtoMessage() {} +func (*Ack) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{2} +} +func (m *Ack) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Ack) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Ack.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Ack) XXX_Merge(src proto.Message) { + xxx_messageInfo_Ack.Merge(m, src) +} +func (m *Ack) XXX_Size() int { + return m.Size() +} +func (m *Ack) XXX_DiscardUnknown() { + xxx_messageInfo_Ack.DiscardUnknown(m) +} + +var xxx_messageInfo_Ack proto.InternalMessageInfo + +func (m *Ack) GetStatus() Status { + if m != nil { + return m.Status + } + return Status_STATUS_UNSPECIFIED +} + +// Both directions after the handshake: admin -> broker is a publication, broker -> +// subscriber a delivery of the same bytes. The single-owner chunk travels as its +// stored chunk data, opaque to the protocol and validated by the ordinary SOC code +// once the `id` slot has been rewritten as the Frames section says: +// id (32) || signature (65) || span (8, LE) || payload (<= 4096) +type Broadcast struct { + Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` +} + +func (m *Broadcast) Reset() { *m = Broadcast{} } +func (m *Broadcast) String() string { return proto.CompactTextString(m) } +func (*Broadcast) ProtoMessage() {} +func (*Broadcast) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{3} +} +func (m *Broadcast) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Broadcast.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Broadcast) XXX_Merge(src proto.Message) { + xxx_messageInfo_Broadcast.Merge(m, src) +} +func (m *Broadcast) XXX_Size() int { + return m.Size() +} +func (m *Broadcast) XXX_DiscardUnknown() { + xxx_messageInfo_Broadcast.DiscardUnknown(m) +} + +var xxx_messageInfo_Broadcast proto.InternalMessageInfo + +func (m *Broadcast) GetSoc() []byte { + if m != nil { + return m.Soc + } + return nil +} + +type Publish struct { + Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` +} + +func (m *Publish) Reset() { *m = Publish{} } +func (m *Publish) String() string { return proto.CompactTextString(m) } +func (*Publish) ProtoMessage() {} +func (*Publish) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{4} +} +func (m *Publish) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Publish) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Publish.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Publish) XXX_Merge(src proto.Message) { + xxx_messageInfo_Publish.Merge(m, src) +} +func (m *Publish) XXX_Size() int { + return m.Size() +} +func (m *Publish) XXX_DiscardUnknown() { + xxx_messageInfo_Publish.DiscardUnknown(m) +} + +var xxx_messageInfo_Publish proto.InternalMessageInfo + +func (m *Publish) GetSoc() []byte { + if m != nil { + return m.Soc + } + return nil +} + +type PayloadMessage struct { + // Types that are valid to be assigned to PayloadMessage: + // *PayloadMessage_Publish + // *PayloadMessage_Broadcast + PayloadMessage isPayloadMessage_PayloadMessage `protobuf_oneof:"payload_message"` +} + +func (m *PayloadMessage) Reset() { *m = PayloadMessage{} } +func (m *PayloadMessage) String() string { return proto.CompactTextString(m) } +func (*PayloadMessage) ProtoMessage() {} +func (*PayloadMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{5} +} +func (m *PayloadMessage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PayloadMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PayloadMessage.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PayloadMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_PayloadMessage.Merge(m, src) +} +func (m *PayloadMessage) XXX_Size() int { + return m.Size() +} +func (m *PayloadMessage) XXX_DiscardUnknown() { + xxx_messageInfo_PayloadMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_PayloadMessage proto.InternalMessageInfo + +type isPayloadMessage_PayloadMessage interface { + isPayloadMessage_PayloadMessage() + MarshalTo([]byte) (int, error) + Size() int +} + +type PayloadMessage_Publish struct { + Publish *Publish `protobuf:"bytes,1,opt,name=publish,proto3,oneof" json:"publish,omitempty"` +} +type PayloadMessage_Broadcast struct { + Broadcast *Broadcast `protobuf:"bytes,2,opt,name=broadcast,proto3,oneof" json:"broadcast,omitempty"` +} + +func (*PayloadMessage_Publish) isPayloadMessage_PayloadMessage() {} +func (*PayloadMessage_Broadcast) isPayloadMessage_PayloadMessage() {} + +func (m *PayloadMessage) GetPayloadMessage() isPayloadMessage_PayloadMessage { + if m != nil { + return m.PayloadMessage + } + return nil +} + +func (m *PayloadMessage) GetPublish() *Publish { + if x, ok := m.GetPayloadMessage().(*PayloadMessage_Publish); ok { + return x.Publish + } + return nil +} + +func (m *PayloadMessage) GetBroadcast() *Broadcast { + if x, ok := m.GetPayloadMessage().(*PayloadMessage_Broadcast); ok { + return x.Broadcast + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*PayloadMessage) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*PayloadMessage_Publish)(nil), + (*PayloadMessage_Broadcast)(nil), + } +} + +type Join struct { + Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` +} + +func (m *Join) Reset() { *m = Join{} } +func (m *Join) String() string { return proto.CompactTextString(m) } +func (*Join) ProtoMessage() {} +func (*Join) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{6} +} +func (m *Join) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Join) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Join.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Join) XXX_Merge(src proto.Message) { + xxx_messageInfo_Join.Merge(m, src) +} +func (m *Join) XXX_Size() int { + return m.Size() +} +func (m *Join) XXX_DiscardUnknown() { + xxx_messageInfo_Join.DiscardUnknown(m) +} + +var xxx_messageInfo_Join proto.InternalMessageInfo + +func (m *Join) GetTopic() []byte { + if m != nil { + return m.Topic + } + return nil +} + +type JoinAck struct { + Challenge []byte `protobuf:"bytes,1,opt,name=challenge,proto3" json:"challenge,omitempty"` +} + +func (m *JoinAck) Reset() { *m = JoinAck{} } +func (m *JoinAck) String() string { return proto.CompactTextString(m) } +func (*JoinAck) ProtoMessage() {} +func (*JoinAck) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{7} +} +func (m *JoinAck) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *JoinAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_JoinAck.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *JoinAck) XXX_Merge(src proto.Message) { + xxx_messageInfo_JoinAck.Merge(m, src) +} +func (m *JoinAck) XXX_Size() int { + return m.Size() +} +func (m *JoinAck) XXX_DiscardUnknown() { + xxx_messageInfo_JoinAck.DiscardUnknown(m) +} + +var xxx_messageInfo_JoinAck proto.InternalMessageInfo + +func (m *JoinAck) GetChallenge() []byte { + if m != nil { + return m.Challenge + } + return nil +} + +type Claim struct { + ChallengeSig []byte `protobuf:"bytes,1,opt,name=challengeSig,proto3" json:"challengeSig,omitempty"` +} + +func (m *Claim) Reset() { *m = Claim{} } +func (m *Claim) String() string { return proto.CompactTextString(m) } +func (*Claim) ProtoMessage() {} +func (*Claim) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{8} +} +func (m *Claim) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Claim) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Claim.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Claim) XXX_Merge(src proto.Message) { + xxx_messageInfo_Claim.Merge(m, src) +} +func (m *Claim) XXX_Size() int { + return m.Size() +} +func (m *Claim) XXX_DiscardUnknown() { + xxx_messageInfo_Claim.DiscardUnknown(m) +} + +var xxx_messageInfo_Claim proto.InternalMessageInfo + +func (m *Claim) GetChallengeSig() []byte { + if m != nil { + return m.ChallengeSig + } + return nil +} + +type SystemMessage struct { + // Types that are valid to be assigned to SysMessage: + // *SystemMessage_Join + // *SystemMessage_Claim + SysMessage isSystemMessage_SysMessage `protobuf_oneof:"sys_message"` +} + +func (m *SystemMessage) Reset() { *m = SystemMessage{} } +func (m *SystemMessage) String() string { return proto.CompactTextString(m) } +func (*SystemMessage) ProtoMessage() {} +func (*SystemMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{9} +} +func (m *SystemMessage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SystemMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SystemMessage.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SystemMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_SystemMessage.Merge(m, src) +} +func (m *SystemMessage) XXX_Size() int { + return m.Size() +} +func (m *SystemMessage) XXX_DiscardUnknown() { + xxx_messageInfo_SystemMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_SystemMessage proto.InternalMessageInfo + +type isSystemMessage_SysMessage interface { + isSystemMessage_SysMessage() + MarshalTo([]byte) (int, error) + Size() int +} + +type SystemMessage_Join struct { + Join *Join `protobuf:"bytes,1,opt,name=join,proto3,oneof" json:"join,omitempty"` +} +type SystemMessage_Claim struct { + Claim *Claim `protobuf:"bytes,2,opt,name=claim,proto3,oneof" json:"claim,omitempty"` +} + +func (*SystemMessage_Join) isSystemMessage_SysMessage() {} +func (*SystemMessage_Claim) isSystemMessage_SysMessage() {} + +func (m *SystemMessage) GetSysMessage() isSystemMessage_SysMessage { + if m != nil { + return m.SysMessage + } + return nil +} + +func (m *SystemMessage) GetJoin() *Join { + if x, ok := m.GetSysMessage().(*SystemMessage_Join); ok { + return x.Join + } + return nil +} + +func (m *SystemMessage) GetClaim() *Claim { + if x, ok := m.GetSysMessage().(*SystemMessage_Claim); ok { + return x.Claim + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*SystemMessage) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*SystemMessage_Join)(nil), + (*SystemMessage_Claim)(nil), + } +} + +func init() { + proto.RegisterEnum("bps.TopicBinding", TopicBinding_name, TopicBinding_value) + proto.RegisterEnum("bps.Status", Status_name, Status_value) + proto.RegisterType((*CohortSpec)(nil), "bps.CohortSpec") + proto.RegisterType((*Auth)(nil), "bps.Auth") + proto.RegisterType((*Ack)(nil), "bps.Ack") + proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") + proto.RegisterType((*Publish)(nil), "bps.Publish") + proto.RegisterType((*PayloadMessage)(nil), "bps.PayloadMessage") + proto.RegisterType((*Join)(nil), "bps.Join") + proto.RegisterType((*JoinAck)(nil), "bps.JoinAck") + proto.RegisterType((*Claim)(nil), "bps.Claim") + proto.RegisterType((*SystemMessage)(nil), "bps.SystemMessage") +} + +func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } + +var fileDescriptor_786299925f8760d5 = []byte{ + // 541 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x53, 0x3d, 0x6f, 0xda, 0x4e, + 0x1c, 0xb6, 0xf1, 0xf9, 0xed, 0x07, 0xe1, 0x7f, 0x39, 0xfd, 0x55, 0xa5, 0x2a, 0x71, 0xa3, 0xeb, + 0xd0, 0x28, 0x91, 0x18, 0xe8, 0xdc, 0x01, 0x83, 0x29, 0xb8, 0x29, 0x41, 0x36, 0x2c, 0x5d, 0x90, + 0x6d, 0x2c, 0x70, 0x0b, 0xb6, 0xc5, 0x99, 0x81, 0xa1, 0x1f, 0xa0, 0x5b, 0x3f, 0x56, 0xc7, 0x8c, + 0x1d, 0x2b, 0xf8, 0x22, 0x95, 0xcf, 0x36, 0x69, 0xa3, 0x6c, 0xbe, 0xdf, 0xf3, 0xdc, 0xf3, 0x72, + 0x77, 0x06, 0xdd, 0x4f, 0x59, 0x3b, 0xdd, 0x26, 0x59, 0x42, 0x24, 0x3f, 0x65, 0xf4, 0xbb, 0x08, + 0xd0, 0x4b, 0x56, 0xc9, 0x36, 0x73, 0xd3, 0x30, 0x20, 0xff, 0x83, 0x9c, 0x25, 0x69, 0x14, 0x5c, + 0x88, 0x57, 0xe2, 0x75, 0xc3, 0x29, 0x16, 0xe4, 0x16, 0x54, 0x3f, 0x8a, 0x17, 0x51, 0xbc, 0xbc, + 0xa8, 0x5d, 0x89, 0xd7, 0xcd, 0xce, 0x79, 0x3b, 0x97, 0x99, 0xe6, 0xa0, 0x59, 0x00, 0x4e, 0xc5, + 0xc8, 0x25, 0xbc, 0xc5, 0x26, 0x8a, 0x2f, 0xe4, 0x42, 0x82, 0x2f, 0x6c, 0xa4, 0x49, 0x18, 0xd9, + 0x48, 0x43, 0x58, 0xb6, 0x91, 0xa6, 0x60, 0xd5, 0x46, 0x9a, 0x8a, 0x35, 0x1b, 0x69, 0x1a, 0xd6, + 0x6d, 0xa4, 0xe9, 0x18, 0xe8, 0x0d, 0xa0, 0xee, 0x2e, 0x5b, 0x91, 0x16, 0xe8, 0x2c, 0x5a, 0xc6, + 0x5e, 0xb6, 0xdb, 0x86, 0x65, 0x90, 0xc7, 0x81, 0x8d, 0xb4, 0x1a, 0x96, 0xa8, 0x09, 0x52, 0x37, + 0xf8, 0x4a, 0xde, 0x80, 0xc2, 0x32, 0x2f, 0xdb, 0x31, 0xce, 0x6b, 0x76, 0xea, 0x3c, 0x98, 0xcb, + 0x47, 0x4e, 0x09, 0x15, 0x3b, 0x9e, 0x24, 0x90, 0xb1, 0x42, 0x2f, 0x41, 0x37, 0xb7, 0x89, 0xb7, + 0x08, 0x3c, 0x96, 0x11, 0x0c, 0x12, 0x4b, 0xaa, 0xde, 0xf9, 0x27, 0x7d, 0x05, 0xea, 0x64, 0xe7, + 0xaf, 0x23, 0xb6, 0x7a, 0x06, 0xfc, 0x06, 0xcd, 0x89, 0xb7, 0x5f, 0x27, 0xde, 0xe2, 0x53, 0xc8, + 0x98, 0xb7, 0x0c, 0xc9, 0x35, 0xa8, 0x69, 0x41, 0xe7, 0xbc, 0x7a, 0xa7, 0xc1, 0xb3, 0x94, 0x12, + 0x43, 0xc1, 0xa9, 0x60, 0xd2, 0x06, 0xdd, 0xaf, 0x7c, 0xf9, 0x81, 0xd6, 0x3b, 0x4d, 0xce, 0x3d, + 0xa5, 0x19, 0x0a, 0xce, 0x23, 0xc5, 0x3c, 0x87, 0xff, 0xd2, 0xc2, 0x6b, 0xbe, 0x29, 0xcc, 0x68, + 0x0b, 0x90, 0x9d, 0x44, 0xf1, 0xf3, 0xf7, 0x45, 0xdf, 0x82, 0x9a, 0xa3, 0xf9, 0x01, 0xb5, 0x40, + 0x0f, 0x56, 0xde, 0x7a, 0x1d, 0xc6, 0xcb, 0xd3, 0x59, 0x9e, 0x06, 0xf4, 0x16, 0xe4, 0xde, 0xda, + 0x8b, 0x36, 0x84, 0x42, 0xe3, 0x34, 0x75, 0xa3, 0x65, 0xc9, 0xfc, 0x67, 0x46, 0x03, 0x38, 0x73, + 0xf7, 0x2c, 0x0b, 0x37, 0x55, 0xe3, 0xd7, 0x80, 0xbe, 0x24, 0x51, 0x5c, 0xd6, 0xd5, 0x79, 0x85, + 0xdc, 0x77, 0x28, 0x38, 0x1c, 0x20, 0x14, 0xe4, 0x20, 0x97, 0x2f, 0x4b, 0x02, 0x67, 0x70, 0xc3, + 0xa1, 0xe0, 0x14, 0x90, 0x79, 0x06, 0x75, 0xb6, 0x67, 0x55, 0xb1, 0x9b, 0xf7, 0xd0, 0xf8, 0xfb, + 0x59, 0x91, 0x4b, 0x78, 0x39, 0xbd, 0x9f, 0x8c, 0x7a, 0x73, 0x73, 0x34, 0xee, 0x8f, 0xc6, 0x1f, + 0xe6, 0xb3, 0xb1, 0x3b, 0xb1, 0x7a, 0xa3, 0xc1, 0xc8, 0xea, 0x63, 0x81, 0x34, 0x01, 0x06, 0x96, + 0xd5, 0x9f, 0x73, 0x0e, 0x46, 0x37, 0x03, 0x50, 0x8a, 0xcb, 0x27, 0x2f, 0x80, 0xb8, 0xd3, 0xee, + 0x74, 0xe6, 0x3e, 0xd9, 0xa1, 0x40, 0xed, 0xfe, 0x23, 0x16, 0x89, 0x06, 0x68, 0x30, 0xbb, 0xbb, + 0xc3, 0x35, 0xd2, 0x00, 0xcd, 0xb1, 0x6c, 0xab, 0x37, 0xb5, 0xfa, 0x18, 0xd1, 0xfc, 0x99, 0x48, + 0x66, 0xeb, 0xe7, 0xc1, 0x10, 0x1f, 0x0e, 0x86, 0xf8, 0xfb, 0x60, 0x88, 0x3f, 0x8e, 0x86, 0xf0, + 0x70, 0x34, 0x84, 0x5f, 0x47, 0x43, 0xf8, 0x5c, 0x4b, 0x7d, 0x5f, 0xe1, 0x3f, 0xd0, 0xbb, 0x3f, + 0x01, 0x00, 0x00, 0xff, 0xff, 0xdc, 0x68, 0x51, 0x2f, 0x4d, 0x03, 0x00, 0x00, +} + +func (m *CohortSpec) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CohortSpec) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CohortSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Admin) > 0 { + i -= len(m.Admin) + copy(dAtA[i:], m.Admin) + i = encodeVarintBps(dAtA, i, uint64(len(m.Admin))) + i-- + dAtA[i] = 0x2a + } + if m.Binding != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x10 + } + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Auth) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Auth) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Auth) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = encodeVarintBps(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Ack) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Ack) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Status != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Status)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Broadcast) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Broadcast) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Soc) > 0 { + i -= len(m.Soc) + copy(dAtA[i:], m.Soc) + i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Publish) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Publish) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Publish) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Soc) > 0 { + i -= len(m.Soc) + copy(dAtA[i:], m.Soc) + i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PayloadMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PayloadMessage) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PayloadMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.PayloadMessage != nil { + { + size := m.PayloadMessage.Size() + i -= size + if _, err := m.PayloadMessage.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *PayloadMessage_Publish) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PayloadMessage_Publish) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Publish != nil { + { + size, err := m.Publish.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBps(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *PayloadMessage_Broadcast) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PayloadMessage_Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Broadcast != nil { + { + size, err := m.Broadcast.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBps(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func (m *Join) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Join) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *JoinAck) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *JoinAck) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *JoinAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Challenge) > 0 { + i -= len(m.Challenge) + copy(dAtA[i:], m.Challenge) + i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Claim) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Claim) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.ChallengeSig) > 0 { + i -= len(m.ChallengeSig) + copy(dAtA[i:], m.ChallengeSig) + i = encodeVarintBps(dAtA, i, uint64(len(m.ChallengeSig))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SystemMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SystemMessage) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SystemMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.SysMessage != nil { + { + size := m.SysMessage.Size() + i -= size + if _, err := m.SysMessage.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *SystemMessage_Join) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SystemMessage_Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Join != nil { + { + size, err := m.Join.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBps(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *SystemMessage_Claim) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SystemMessage_Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Claim != nil { + { + size, err := m.Claim.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBps(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func encodeVarintBps(dAtA []byte, offset int, v uint64) int { + offset -= sovBps(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *CohortSpec) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Topic) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Binding != 0 { + n += 1 + sovBps(uint64(m.Binding)) + } + l = len(m.Admin) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *Auth) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Signature) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *Ack) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != 0 { + n += 1 + sovBps(uint64(m.Status)) + } + return n +} + +func (m *Broadcast) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Soc) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *Publish) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Soc) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *PayloadMessage) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.PayloadMessage != nil { + n += m.PayloadMessage.Size() + } + return n +} + +func (m *PayloadMessage_Publish) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Publish != nil { + l = m.Publish.Size() + n += 1 + l + sovBps(uint64(l)) + } + return n +} +func (m *PayloadMessage_Broadcast) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Broadcast != nil { + l = m.Broadcast.Size() + n += 1 + l + sovBps(uint64(l)) + } + return n +} +func (m *Join) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Topic) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *JoinAck) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Challenge) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *Claim) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ChallengeSig) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *SystemMessage) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.SysMessage != nil { + n += m.SysMessage.Size() + } + return n +} + +func (m *SystemMessage_Join) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Join != nil { + l = m.Join.Size() + n += 1 + l + sovBps(uint64(l)) + } + return n +} +func (m *SystemMessage_Claim) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Claim != nil { + l = m.Claim.Size() + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func sovBps(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozBps(x uint64) (n int) { + return sovBps(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *CohortSpec) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CohortSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CohortSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) + if m.Topic == nil { + m.Topic = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= TopicBinding(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Admin", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Admin = append(m.Admin[:0], dAtA[iNdEx:postIndex]...) + if m.Admin == nil { + m.Admin = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Auth) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Auth: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Auth: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) + if m.Signature == nil { + m.Signature = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Ack) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Ack: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Ack: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= Status(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Broadcast) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Broadcast: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Broadcast: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) + if m.Soc == nil { + m.Soc = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Publish) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Publish: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Publish: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) + if m.Soc == nil { + m.Soc = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PayloadMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PayloadMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PayloadMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Publish", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &Publish{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.PayloadMessage = &PayloadMessage_Publish{v} + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Broadcast", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &Broadcast{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.PayloadMessage = &PayloadMessage_Broadcast{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Join) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Join: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Join: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) + if m.Topic == nil { + m.Topic = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *JoinAck) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: JoinAck: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: JoinAck: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Claim) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Claim: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Claim: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ChallengeSig", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ChallengeSig = append(m.ChallengeSig[:0], dAtA[iNdEx:postIndex]...) + if m.ChallengeSig == nil { + m.ChallengeSig = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SystemMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SystemMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SystemMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Join", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &Join{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.SysMessage = &SystemMessage_Join{v} + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Claim", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &Claim{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.SysMessage = &SystemMessage_Claim{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipBps(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBps + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBps + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBps + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthBps + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupBps + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthBps + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthBps = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowBps = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupBps = fmt.Errorf("proto: unexpected end of group") +) diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto new file mode 100644 index 00000000000..e7efd3d4567 --- /dev/null +++ b/pkg/bps/pb/bps.proto @@ -0,0 +1,103 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +syntax = "proto3"; + +package bps; + +option go_package = "pb"; + + + +// What the topic binds to. One binding. +enum TopicBinding { + TOPIC_BINDING_UNSPECIFIED = 0; // invalid on the wire + FEED_TOPIC = 4; // signed id = keccak256(topic || index). SWIP-60's + // number; 1, 2, 3, 5 are SWIP-60's other bindings +} + +// The cohort's identity. Fixed by whoever joins first; immutable. +message CohortSpec { + bytes topic = 1; // 32 bytes + TopicBinding binding = 2; // FEED_TOPIC + bytes admin = 5; // 20 bytes, required + reserved 3, 4, 6, 7, 8, 9; // SWIP-60: publishers, history, (unused), (unused), + // (unused), spectators — none apply here +} + +// Proof of an identity, bound to the stream it arrives on: +// owner = ecrecover( keccak256("bps-join:v1" || topic || admin), signature ) +// The preimage is static and carries no node identity (SWIP-60): the same key works +// from any node, and a join never links an address to a peer id. It is therefore +// replayable, and a replayed identity is worthless: authorship rests on the message +// signature, never on the handshake. +message Auth { + bytes signature = 1; // 65 bytes + reserved 2; // SWIP-60: id, for bindings that do not fix it +} + +// Peer -> broker: the first and only handshake frame on a fresh stream. Creates the +// cohort if no live cohort has this spec, attaches to it otherwise. `auth` binds an +// identity to THIS stream; absent, the stream has none and is read-only. +//message Join { + //CohortSpec cohort = 1; + //Auth auth = 2; +//} + +enum Status { + STATUS_UNSPECIFIED = 0; // invalid on the wire + OK = 1; + FULL = 2; // a capacity bound (see Resource bounds) + REJECTED = 4; // spec outside this SWIP, or a reserved field set + reserved 3; // SWIP-60: UNKNOWN_TOPIC — cannot occur: Join creates +} + +// Broker -> peer, answering Join. Status only. A non-OK Ack ends the stream. +message Ack { + Status status = 1; + reserved 2, 3, 4, 5; // SWIP-60: spec echo and service SOCs — nothing to echo +} + +// Both directions after the handshake: admin -> broker is a publication, broker -> +// subscriber a delivery of the same bytes. The single-owner chunk travels as its +// stored chunk data, opaque to the protocol and validated by the ordinary SOC code +// once the `id` slot has been rewritten as the Frames section says: +// id (32) || signature (65) || span (8, LE) || payload (<= 4096) +message Broadcast { + bytes soc = 1; +} + +message Publish { + bytes soc = 1; +} + +message PayloadMessage { + oneof payload_message { + Publish publish = 1; + Broadcast broadcast = 2; + + } +} + +message Join { + bytes topic = 1; +} + +message JoinAck { + bytes challenge = 1; // the challenge to sign in order to claim the cohort's publishing role stream +} + +message Claim { + bytes challengeSig = 1; // the challenge signature that takes over the cohort. signs the challenge in JoinAck. +} + + +message SystemMessage { + oneof sys_message { + Join join = 1; + Claim claim =2; + } +} + + diff --git a/pkg/bps/pb/doc.go b/pkg/bps/pb/doc.go new file mode 100644 index 00000000000..57b18ce87e9 --- /dev/null +++ b/pkg/bps/pb/doc.go @@ -0,0 +1,8 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate sh -c "protoc -I . -I \"$(go list -f '{{ .Dir }}' -m github.com/gogo/protobuf)/protobuf\" --gogofaster_out=. bps.proto" + +// Package pb holds only Protocol Buffer definitions and generated code. +package pb diff --git a/pkg/bps/registry.go b/pkg/bps/registry.go new file mode 100644 index 00000000000..bbf96642fa9 --- /dev/null +++ b/pkg/bps/registry.go @@ -0,0 +1,33 @@ +package bps + +import ( + "sync" + + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +func NewRegistry() *CohortRegistry { + return &CohortRegistry{ + cohorts: make(map[string]*Cohort), + } +} + +type CohortRegistry struct { + mtx sync.Mutex + cohorts map[string]*Cohort +} + +// Jopen is a join/open operation in one - the peer either joins an existing +// cohort or creates one by joining a previously unknown topic. +func (c *CohortRegistry) Jopen(overlay swarm.Address, topic []byte) *Cohort { + return nil +} + +type Cohort struct { + mtx sync.Mutex + topic []byte // topic is the feed topic + owner hash + lastSeen uint64 // last feed update index, used to prevent replay and circumvent dedup logic (for now) + members map[string]string + publisher swarm.Address + challenge []byte +} diff --git a/pkg/bps/registry_test.go b/pkg/bps/registry_test.go new file mode 100644 index 00000000000..7c30ef52b76 --- /dev/null +++ b/pkg/bps/registry_test.go @@ -0,0 +1 @@ +package bps_test From 061e0408da2277e98ff250f7484796a4844d4d75 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:01:49 -0600 Subject: [PATCH 02/29] chore: strip protobuf (cherry picked from commit 3a8283ddd9c846b1aee0812812e11936aad097ea) --- pkg/bps/pb/bps.pb.go | 732 ++----------------------------------------- pkg/bps/pb/bps.proto | 49 --- 2 files changed, 28 insertions(+), 753 deletions(-) diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index c426d596f57..a0d79961608 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -22,219 +22,6 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package -// What the topic binds to. One binding. -type TopicBinding int32 - -const ( - TopicBinding_TOPIC_BINDING_UNSPECIFIED TopicBinding = 0 - TopicBinding_FEED_TOPIC TopicBinding = 4 -) - -var TopicBinding_name = map[int32]string{ - 0: "TOPIC_BINDING_UNSPECIFIED", - 4: "FEED_TOPIC", -} - -var TopicBinding_value = map[string]int32{ - "TOPIC_BINDING_UNSPECIFIED": 0, - "FEED_TOPIC": 4, -} - -func (x TopicBinding) String() string { - return proto.EnumName(TopicBinding_name, int32(x)) -} - -func (TopicBinding) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{0} -} - -type Status int32 - -const ( - Status_STATUS_UNSPECIFIED Status = 0 - Status_OK Status = 1 - Status_FULL Status = 2 - Status_REJECTED Status = 4 -) - -var Status_name = map[int32]string{ - 0: "STATUS_UNSPECIFIED", - 1: "OK", - 2: "FULL", - 4: "REJECTED", -} - -var Status_value = map[string]int32{ - "STATUS_UNSPECIFIED": 0, - "OK": 1, - "FULL": 2, - "REJECTED": 4, -} - -func (x Status) String() string { - return proto.EnumName(Status_name, int32(x)) -} - -func (Status) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{1} -} - -// The cohort's identity. Fixed by whoever joins first; immutable. -type CohortSpec struct { - Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` - Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` - Admin []byte `protobuf:"bytes,5,opt,name=admin,proto3" json:"admin,omitempty"` -} - -func (m *CohortSpec) Reset() { *m = CohortSpec{} } -func (m *CohortSpec) String() string { return proto.CompactTextString(m) } -func (*CohortSpec) ProtoMessage() {} -func (*CohortSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{0} -} -func (m *CohortSpec) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CohortSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CohortSpec.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *CohortSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_CohortSpec.Merge(m, src) -} -func (m *CohortSpec) XXX_Size() int { - return m.Size() -} -func (m *CohortSpec) XXX_DiscardUnknown() { - xxx_messageInfo_CohortSpec.DiscardUnknown(m) -} - -var xxx_messageInfo_CohortSpec proto.InternalMessageInfo - -func (m *CohortSpec) GetTopic() []byte { - if m != nil { - return m.Topic - } - return nil -} - -func (m *CohortSpec) GetBinding() TopicBinding { - if m != nil { - return m.Binding - } - return TopicBinding_TOPIC_BINDING_UNSPECIFIED -} - -func (m *CohortSpec) GetAdmin() []byte { - if m != nil { - return m.Admin - } - return nil -} - -// Proof of an identity, bound to the stream it arrives on: -// owner = ecrecover( keccak256("bps-join:v1" || topic || admin), signature ) -// The preimage is static and carries no node identity (SWIP-60): the same key works -// from any node, and a join never links an address to a peer id. It is therefore -// replayable, and a replayed identity is worthless: authorship rests on the message -// signature, never on the handshake. -type Auth struct { - Signature []byte `protobuf:"bytes,1,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (m *Auth) Reset() { *m = Auth{} } -func (m *Auth) String() string { return proto.CompactTextString(m) } -func (*Auth) ProtoMessage() {} -func (*Auth) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{1} -} -func (m *Auth) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Auth) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Auth.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Auth) XXX_Merge(src proto.Message) { - xxx_messageInfo_Auth.Merge(m, src) -} -func (m *Auth) XXX_Size() int { - return m.Size() -} -func (m *Auth) XXX_DiscardUnknown() { - xxx_messageInfo_Auth.DiscardUnknown(m) -} - -var xxx_messageInfo_Auth proto.InternalMessageInfo - -func (m *Auth) GetSignature() []byte { - if m != nil { - return m.Signature - } - return nil -} - -// Broker -> peer, answering Join. Status only. A non-OK Ack ends the stream. -type Ack struct { - Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` -} - -func (m *Ack) Reset() { *m = Ack{} } -func (m *Ack) String() string { return proto.CompactTextString(m) } -func (*Ack) ProtoMessage() {} -func (*Ack) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{2} -} -func (m *Ack) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Ack) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Ack.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Ack) XXX_Merge(src proto.Message) { - xxx_messageInfo_Ack.Merge(m, src) -} -func (m *Ack) XXX_Size() int { - return m.Size() -} -func (m *Ack) XXX_DiscardUnknown() { - xxx_messageInfo_Ack.DiscardUnknown(m) -} - -var xxx_messageInfo_Ack proto.InternalMessageInfo - -func (m *Ack) GetStatus() Status { - if m != nil { - return m.Status - } - return Status_STATUS_UNSPECIFIED -} - // Both directions after the handshake: admin -> broker is a publication, broker -> // subscriber a delivery of the same bytes. The single-owner chunk travels as its // stored chunk data, opaque to the protocol and validated by the ordinary SOC code @@ -248,7 +35,7 @@ func (m *Broadcast) Reset() { *m = Broadcast{} } func (m *Broadcast) String() string { return proto.CompactTextString(m) } func (*Broadcast) ProtoMessage() {} func (*Broadcast) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{3} + return fileDescriptor_786299925f8760d5, []int{0} } func (m *Broadcast) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -292,7 +79,7 @@ func (m *Publish) Reset() { *m = Publish{} } func (m *Publish) String() string { return proto.CompactTextString(m) } func (*Publish) ProtoMessage() {} func (*Publish) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{4} + return fileDescriptor_786299925f8760d5, []int{1} } func (m *Publish) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -339,7 +126,7 @@ func (m *PayloadMessage) Reset() { *m = PayloadMessage{} } func (m *PayloadMessage) String() string { return proto.CompactTextString(m) } func (*PayloadMessage) ProtoMessage() {} func (*PayloadMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{5} + return fileDescriptor_786299925f8760d5, []int{2} } func (m *PayloadMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -421,7 +208,7 @@ func (m *Join) Reset() { *m = Join{} } func (m *Join) String() string { return proto.CompactTextString(m) } func (*Join) ProtoMessage() {} func (*Join) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{6} + return fileDescriptor_786299925f8760d5, []int{3} } func (m *Join) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -465,7 +252,7 @@ func (m *JoinAck) Reset() { *m = JoinAck{} } func (m *JoinAck) String() string { return proto.CompactTextString(m) } func (*JoinAck) ProtoMessage() {} func (*JoinAck) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{7} + return fileDescriptor_786299925f8760d5, []int{4} } func (m *JoinAck) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -509,7 +296,7 @@ func (m *Claim) Reset() { *m = Claim{} } func (m *Claim) String() string { return proto.CompactTextString(m) } func (*Claim) ProtoMessage() {} func (*Claim) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{8} + return fileDescriptor_786299925f8760d5, []int{5} } func (m *Claim) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -556,7 +343,7 @@ func (m *SystemMessage) Reset() { *m = SystemMessage{} } func (m *SystemMessage) String() string { return proto.CompactTextString(m) } func (*SystemMessage) ProtoMessage() {} func (*SystemMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{9} + return fileDescriptor_786299925f8760d5, []int{6} } func (m *SystemMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -631,11 +418,6 @@ func (*SystemMessage) XXX_OneofWrappers() []interface{} { } func init() { - proto.RegisterEnum("bps.TopicBinding", TopicBinding_name, TopicBinding_value) - proto.RegisterEnum("bps.Status", Status_name, Status_value) - proto.RegisterType((*CohortSpec)(nil), "bps.CohortSpec") - proto.RegisterType((*Auth)(nil), "bps.Auth") - proto.RegisterType((*Ack)(nil), "bps.Ack") proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") proto.RegisterType((*Publish)(nil), "bps.Publish") proto.RegisterType((*PayloadMessage)(nil), "bps.PayloadMessage") @@ -648,141 +430,27 @@ func init() { func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 541 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x53, 0x3d, 0x6f, 0xda, 0x4e, - 0x1c, 0xb6, 0xf1, 0xf9, 0xed, 0x07, 0xe1, 0x7f, 0x39, 0xfd, 0x55, 0xa5, 0x2a, 0x71, 0xa3, 0xeb, - 0xd0, 0x28, 0x91, 0x18, 0xe8, 0xdc, 0x01, 0x83, 0x29, 0xb8, 0x29, 0x41, 0x36, 0x2c, 0x5d, 0x90, - 0x6d, 0x2c, 0x70, 0x0b, 0xb6, 0xc5, 0x99, 0x81, 0xa1, 0x1f, 0xa0, 0x5b, 0x3f, 0x56, 0xc7, 0x8c, - 0x1d, 0x2b, 0xf8, 0x22, 0x95, 0xcf, 0x36, 0x69, 0xa3, 0x6c, 0xbe, 0xdf, 0xf3, 0xdc, 0xf3, 0x72, - 0x77, 0x06, 0xdd, 0x4f, 0x59, 0x3b, 0xdd, 0x26, 0x59, 0x42, 0x24, 0x3f, 0x65, 0xf4, 0xbb, 0x08, - 0xd0, 0x4b, 0x56, 0xc9, 0x36, 0x73, 0xd3, 0x30, 0x20, 0xff, 0x83, 0x9c, 0x25, 0x69, 0x14, 0x5c, - 0x88, 0x57, 0xe2, 0x75, 0xc3, 0x29, 0x16, 0xe4, 0x16, 0x54, 0x3f, 0x8a, 0x17, 0x51, 0xbc, 0xbc, - 0xa8, 0x5d, 0x89, 0xd7, 0xcd, 0xce, 0x79, 0x3b, 0x97, 0x99, 0xe6, 0xa0, 0x59, 0x00, 0x4e, 0xc5, - 0xc8, 0x25, 0xbc, 0xc5, 0x26, 0x8a, 0x2f, 0xe4, 0x42, 0x82, 0x2f, 0x6c, 0xa4, 0x49, 0x18, 0xd9, - 0x48, 0x43, 0x58, 0xb6, 0x91, 0xa6, 0x60, 0xd5, 0x46, 0x9a, 0x8a, 0x35, 0x1b, 0x69, 0x1a, 0xd6, - 0x6d, 0xa4, 0xe9, 0x18, 0xe8, 0x0d, 0xa0, 0xee, 0x2e, 0x5b, 0x91, 0x16, 0xe8, 0x2c, 0x5a, 0xc6, - 0x5e, 0xb6, 0xdb, 0x86, 0x65, 0x90, 0xc7, 0x81, 0x8d, 0xb4, 0x1a, 0x96, 0xa8, 0x09, 0x52, 0x37, - 0xf8, 0x4a, 0xde, 0x80, 0xc2, 0x32, 0x2f, 0xdb, 0x31, 0xce, 0x6b, 0x76, 0xea, 0x3c, 0x98, 0xcb, - 0x47, 0x4e, 0x09, 0x15, 0x3b, 0x9e, 0x24, 0x90, 0xb1, 0x42, 0x2f, 0x41, 0x37, 0xb7, 0x89, 0xb7, - 0x08, 0x3c, 0x96, 0x11, 0x0c, 0x12, 0x4b, 0xaa, 0xde, 0xf9, 0x27, 0x7d, 0x05, 0xea, 0x64, 0xe7, - 0xaf, 0x23, 0xb6, 0x7a, 0x06, 0xfc, 0x06, 0xcd, 0x89, 0xb7, 0x5f, 0x27, 0xde, 0xe2, 0x53, 0xc8, - 0x98, 0xb7, 0x0c, 0xc9, 0x35, 0xa8, 0x69, 0x41, 0xe7, 0xbc, 0x7a, 0xa7, 0xc1, 0xb3, 0x94, 0x12, - 0x43, 0xc1, 0xa9, 0x60, 0xd2, 0x06, 0xdd, 0xaf, 0x7c, 0xf9, 0x81, 0xd6, 0x3b, 0x4d, 0xce, 0x3d, - 0xa5, 0x19, 0x0a, 0xce, 0x23, 0xc5, 0x3c, 0x87, 0xff, 0xd2, 0xc2, 0x6b, 0xbe, 0x29, 0xcc, 0x68, - 0x0b, 0x90, 0x9d, 0x44, 0xf1, 0xf3, 0xf7, 0x45, 0xdf, 0x82, 0x9a, 0xa3, 0xf9, 0x01, 0xb5, 0x40, - 0x0f, 0x56, 0xde, 0x7a, 0x1d, 0xc6, 0xcb, 0xd3, 0x59, 0x9e, 0x06, 0xf4, 0x16, 0xe4, 0xde, 0xda, - 0x8b, 0x36, 0x84, 0x42, 0xe3, 0x34, 0x75, 0xa3, 0x65, 0xc9, 0xfc, 0x67, 0x46, 0x03, 0x38, 0x73, - 0xf7, 0x2c, 0x0b, 0x37, 0x55, 0xe3, 0xd7, 0x80, 0xbe, 0x24, 0x51, 0x5c, 0xd6, 0xd5, 0x79, 0x85, - 0xdc, 0x77, 0x28, 0x38, 0x1c, 0x20, 0x14, 0xe4, 0x20, 0x97, 0x2f, 0x4b, 0x02, 0x67, 0x70, 0xc3, - 0xa1, 0xe0, 0x14, 0x90, 0x79, 0x06, 0x75, 0xb6, 0x67, 0x55, 0xb1, 0x9b, 0xf7, 0xd0, 0xf8, 0xfb, - 0x59, 0x91, 0x4b, 0x78, 0x39, 0xbd, 0x9f, 0x8c, 0x7a, 0x73, 0x73, 0x34, 0xee, 0x8f, 0xc6, 0x1f, - 0xe6, 0xb3, 0xb1, 0x3b, 0xb1, 0x7a, 0xa3, 0xc1, 0xc8, 0xea, 0x63, 0x81, 0x34, 0x01, 0x06, 0x96, - 0xd5, 0x9f, 0x73, 0x0e, 0x46, 0x37, 0x03, 0x50, 0x8a, 0xcb, 0x27, 0x2f, 0x80, 0xb8, 0xd3, 0xee, - 0x74, 0xe6, 0x3e, 0xd9, 0xa1, 0x40, 0xed, 0xfe, 0x23, 0x16, 0x89, 0x06, 0x68, 0x30, 0xbb, 0xbb, - 0xc3, 0x35, 0xd2, 0x00, 0xcd, 0xb1, 0x6c, 0xab, 0x37, 0xb5, 0xfa, 0x18, 0xd1, 0xfc, 0x99, 0x48, - 0x66, 0xeb, 0xe7, 0xc1, 0x10, 0x1f, 0x0e, 0x86, 0xf8, 0xfb, 0x60, 0x88, 0x3f, 0x8e, 0x86, 0xf0, - 0x70, 0x34, 0x84, 0x5f, 0x47, 0x43, 0xf8, 0x5c, 0x4b, 0x7d, 0x5f, 0xe1, 0x3f, 0xd0, 0xbb, 0x3f, - 0x01, 0x00, 0x00, 0xff, 0xff, 0xdc, 0x68, 0x51, 0x2f, 0x4d, 0x03, 0x00, 0x00, -} - -func (m *CohortSpec) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *CohortSpec) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *CohortSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Admin) > 0 { - i -= len(m.Admin) - copy(dAtA[i:], m.Admin) - i = encodeVarintBps(dAtA, i, uint64(len(m.Admin))) - i-- - dAtA[i] = 0x2a - } - if m.Binding != 0 { - i = encodeVarintBps(dAtA, i, uint64(m.Binding)) - i-- - dAtA[i] = 0x10 - } - if len(m.Topic) > 0 { - i -= len(m.Topic) - copy(dAtA[i:], m.Topic) - i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *Auth) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Auth) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Auth) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Signature) > 0 { - i -= len(m.Signature) - copy(dAtA[i:], m.Signature) - i = encodeVarintBps(dAtA, i, uint64(len(m.Signature))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *Ack) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Ack) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Status != 0 { - i = encodeVarintBps(dAtA, i, uint64(m.Status)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil + // 311 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcb, 0x4a, 0xc3, 0x40, + 0x14, 0x86, 0x27, 0xbd, 0x58, 0x72, 0x7a, 0x51, 0x07, 0x17, 0x05, 0xeb, 0x28, 0xb3, 0xb1, 0x20, + 0x74, 0xa1, 0x4f, 0x60, 0xdc, 0x04, 0x41, 0x28, 0xe9, 0xce, 0x8d, 0xcc, 0x4c, 0x43, 0x3a, 0x9a, + 0x64, 0x86, 0x4e, 0x5c, 0x64, 0xe1, 0x3b, 0xf8, 0x58, 0x2e, 0xbb, 0x74, 0x29, 0xc9, 0x8b, 0x48, + 0x26, 0x49, 0x8b, 0xe0, 0x2e, 0x39, 0xff, 0xc7, 0xf9, 0xce, 0xcf, 0x80, 0xcb, 0xb5, 0x59, 0xe8, + 0xad, 0xca, 0x14, 0xee, 0x72, 0x6d, 0xe8, 0x05, 0xb8, 0xde, 0x56, 0xb1, 0xb5, 0x60, 0x26, 0xc3, + 0x27, 0xd0, 0x35, 0x4a, 0x4c, 0x9d, 0x2b, 0x67, 0x3e, 0x0a, 0xaa, 0x4f, 0x7a, 0x0e, 0x83, 0xe5, + 0x3b, 0x8f, 0xa5, 0xd9, 0xfc, 0x13, 0x7e, 0xc0, 0x64, 0xc9, 0xf2, 0x58, 0xb1, 0xf5, 0x53, 0x68, + 0x0c, 0x8b, 0x42, 0x3c, 0x87, 0x81, 0xae, 0x71, 0xcb, 0x0d, 0x6f, 0x47, 0x8b, 0xca, 0xd7, 0xac, + 0xf0, 0x51, 0xd0, 0xc6, 0x78, 0x01, 0x2e, 0x6f, 0xbd, 0xd3, 0x8e, 0x65, 0x27, 0x96, 0xdd, 0x5f, + 0xe3, 0xa3, 0xe0, 0x80, 0x78, 0xa7, 0x70, 0xac, 0x6b, 0xd7, 0x4b, 0x52, 0xcb, 0xe8, 0x0c, 0x7a, + 0x8f, 0x4a, 0xa6, 0xf8, 0x0c, 0xfa, 0x99, 0xd2, 0xb2, 0x3d, 0xad, 0xfe, 0xa1, 0xd7, 0x30, 0xa8, + 0xd2, 0x7b, 0xf1, 0x86, 0x67, 0xe0, 0x8a, 0x0d, 0x8b, 0xe3, 0x30, 0x8d, 0xc2, 0x06, 0x3a, 0x0c, + 0xe8, 0x0d, 0xf4, 0x1f, 0x62, 0x26, 0x13, 0x4c, 0x61, 0xb4, 0x9f, 0xae, 0x64, 0xd4, 0x90, 0x7f, + 0x66, 0x54, 0xc0, 0x78, 0x95, 0x9b, 0x2c, 0x4c, 0xda, 0xc6, 0x97, 0xd0, 0x7b, 0x55, 0x32, 0x6d, + 0xea, 0xba, 0xb6, 0x42, 0xe5, 0xf5, 0x51, 0x60, 0x03, 0x4c, 0xa1, 0x2f, 0xaa, 0xf5, 0x4d, 0x49, + 0xb0, 0x84, 0x15, 0xfa, 0x28, 0xa8, 0x23, 0x6f, 0x0c, 0x43, 0x93, 0x9b, 0xb6, 0x98, 0x37, 0xfb, + 0x2a, 0x88, 0xb3, 0x2b, 0x88, 0xf3, 0x53, 0x10, 0xe7, 0xb3, 0x24, 0x68, 0x57, 0x12, 0xf4, 0x5d, + 0x12, 0xf4, 0xdc, 0xd1, 0x9c, 0x1f, 0xd9, 0xd7, 0xbb, 0xfb, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x59, + 0x6f, 0x46, 0x7c, 0xca, 0x01, 0x00, 0x00, } func (m *Broadcast) Marshal() (dAtA []byte, err error) { @@ -1094,51 +762,6 @@ func encodeVarintBps(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *CohortSpec) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Topic) - if l > 0 { - n += 1 + l + sovBps(uint64(l)) - } - if m.Binding != 0 { - n += 1 + sovBps(uint64(m.Binding)) - } - l = len(m.Admin) - if l > 0 { - n += 1 + l + sovBps(uint64(l)) - } - return n -} - -func (m *Auth) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Signature) - if l > 0 { - n += 1 + l + sovBps(uint64(l)) - } - return n -} - -func (m *Ack) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Status != 0 { - n += 1 + sovBps(uint64(m.Status)) - } - return n -} - func (m *Broadcast) Size() (n int) { if m == nil { return 0 @@ -1283,305 +906,6 @@ func sovBps(x uint64) (n int) { func sozBps(x uint64) (n int) { return sovBps(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *CohortSpec) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: CohortSpec: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: CohortSpec: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) - if m.Topic == nil { - m.Topic = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) - } - m.Binding = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Binding |= TopicBinding(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Admin", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Admin = append(m.Admin[:0], dAtA[iNdEx:postIndex]...) - if m.Admin == nil { - m.Admin = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipBps(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Auth) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Auth: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Auth: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) - if m.Signature == nil { - m.Signature = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipBps(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Ack) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Ack: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Ack: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) - } - m.Status = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Status |= Status(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipBps(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *Broadcast) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index e7efd3d4567..243d42846cb 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -10,54 +10,6 @@ option go_package = "pb"; -// What the topic binds to. One binding. -enum TopicBinding { - TOPIC_BINDING_UNSPECIFIED = 0; // invalid on the wire - FEED_TOPIC = 4; // signed id = keccak256(topic || index). SWIP-60's - // number; 1, 2, 3, 5 are SWIP-60's other bindings -} - -// The cohort's identity. Fixed by whoever joins first; immutable. -message CohortSpec { - bytes topic = 1; // 32 bytes - TopicBinding binding = 2; // FEED_TOPIC - bytes admin = 5; // 20 bytes, required - reserved 3, 4, 6, 7, 8, 9; // SWIP-60: publishers, history, (unused), (unused), - // (unused), spectators — none apply here -} - -// Proof of an identity, bound to the stream it arrives on: -// owner = ecrecover( keccak256("bps-join:v1" || topic || admin), signature ) -// The preimage is static and carries no node identity (SWIP-60): the same key works -// from any node, and a join never links an address to a peer id. It is therefore -// replayable, and a replayed identity is worthless: authorship rests on the message -// signature, never on the handshake. -message Auth { - bytes signature = 1; // 65 bytes - reserved 2; // SWIP-60: id, for bindings that do not fix it -} - -// Peer -> broker: the first and only handshake frame on a fresh stream. Creates the -// cohort if no live cohort has this spec, attaches to it otherwise. `auth` binds an -// identity to THIS stream; absent, the stream has none and is read-only. -//message Join { - //CohortSpec cohort = 1; - //Auth auth = 2; -//} - -enum Status { - STATUS_UNSPECIFIED = 0; // invalid on the wire - OK = 1; - FULL = 2; // a capacity bound (see Resource bounds) - REJECTED = 4; // spec outside this SWIP, or a reserved field set - reserved 3; // SWIP-60: UNKNOWN_TOPIC — cannot occur: Join creates -} - -// Broker -> peer, answering Join. Status only. A non-OK Ack ends the stream. -message Ack { - Status status = 1; - reserved 2, 3, 4, 5; // SWIP-60: spec echo and service SOCs — nothing to echo -} // Both directions after the handshake: admin -> broker is a publication, broker -> // subscriber a delivery of the same bytes. The single-owner chunk travels as its @@ -76,7 +28,6 @@ message PayloadMessage { oneof payload_message { Publish publish = 1; Broadcast broadcast = 2; - } } From b138342b89517901a2e9697c8671ef9906b99d10 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:16:52 -0600 Subject: [PATCH 03/29] chore: improve structure --- pkg/bps/bps.go | 147 ++++++----- pkg/bps/bps_test.go | 40 +-- pkg/bps/pb/bps.pb.go | 608 +++++++++---------------------------------- pkg/bps/pb/bps.proto | 15 +- 4 files changed, 227 insertions(+), 583 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 9124020c9db..ee80a566e90 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -22,17 +22,16 @@ import ( const loggerName = "bps" const ( - protocolName = "bps" - protocolVersion = "1.0.0" - streamNameSystem = "bps-system" - streamNameMessage = "bps-message" + protocolName = "bps" + protocolVersion = "1.0.0" + streamName = "bps" ) // Interface is the main interface of the bps protocol. -type Interface interface { - Join(ctx context.Context, p p2p.Peer, topic []byte, notify <-chan []byte) error - Publish(ctx context.Context, topic []byte, message []byte) error -} +//type Interface interface { +//Join(ctx context.Context, p p2p.Peer, topic []byte, notify <-chan []byte) error +//Publish(ctx context.Context, topic []byte, message []byte) error +//} // Service is the bps protocol service. type Service struct { @@ -41,9 +40,6 @@ type Service struct { // cohort registry - keep track of members, publisher, channels, challenge, streams? registry *CohortRegistry - // joined - joined map[string]chan []byte - logger log.Logger } @@ -52,7 +48,6 @@ func New(streamer p2p.Streamer, logger log.Logger) *Service { return &Service{ streamer: streamer, registry: NewRegistry(), - joined: make(map[string]chan []byte), logger: logger.WithName(loggerName).Register(), } } @@ -64,11 +59,8 @@ func (s *Service) Protocol() p2p.ProtocolSpec { Version: protocolVersion, StreamSpecs: []p2p.StreamSpec{ { - Name: streamNameSystem, - Handler: s.handlerSystem, - }, { - Name: streamNameMessage, - Handler: s.handlerMessage, + Name: streamName, + Handler: s.handler, }, }, } @@ -76,15 +68,12 @@ func (s *Service) Protocol() p2p.ProtocolSpec { // Join onto a topic at the remote peer. The channel passed at registration will be notified // with the payloads once they arrive. Returns the challenge and a channel that would send -// the payloads over it +// the payloads over it. func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) ([]byte, chan []byte, error) { - stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamNameSystem) + stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) if err != nil { return nil, nil, fmt.Errorf("new stream: %w", err) } - defer func() { - go stream.FullClose() - }() w, r := protobuf.NewWriterAndReader(stream) joinMsg := pb.SystemMessage{SysMessage: &pb.SystemMessage_Join{ @@ -97,62 +86,98 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) if err := r.ReadMsgWithContext(ctx, &joinAck); err != nil { return nil, nil, fmt.Errorf("read join ack: %w", err) } + rxCh := make(chan []byte) - s.joined[string(topic)] = rxCh - return joinAck.Challenge, rxCh, nil -} -func (s *Service) handlerSystem(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { - _, r := protobuf.NewWriterAndReader(stream) - defer stream.FullClose() + // from now on we expect only to read updates off this stream + go func() { + defer stream.FullClose() - var sysMsg pb.SystemMessage - if err := r.ReadMsgWithContext(ctx, &sysMsg); err != nil { - return fmt.Errorf("read sys message: %w", err) - } + for { + msg := pb.Broadcast{} + if err := r.ReadMsgWithContext(ctx, &msg); err != nil { + s.logger.Error(err, "read join ack") + return + } - if join := sysMsg.GetJoin(); join != nil { - // peer is trying to join the cohort. accept and return the challenge + // we might want to do some input validation to see that the broker isn't tricking us - return nil + select { + case rxCh <- msg.Soc: + default: + } + + } + }() + return joinAck.Challenge, rxCh, nil +} + +// claim a given topic on a broker with the provided signature. returns the write channel used in order +// to send later payloads +func (s *Service) Claim(ctx context.Context, address swarm.Address, topic, sig []byte) (chan []byte, error) { + stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) + if err != nil { + return nil, fmt.Errorf("new stream: %w", err) } - if claim := sysMsg.GetClaim(); claim != nil { - // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off - return nil + w, r := protobuf.NewWriterAndReader(stream) + claim := pb.SystemMessage{SysMessage: &pb.SystemMessage_Claim{ + Claim: &pb.Claim{Sig: topic}, + }} + if err := w.WriteMsgWithContext(ctx, &claim); err != nil { + return nil, fmt.Errorf("write claim: %w", err) + } + claimAck := pb.ClaimAck{} + // if the claim is wrong we will just get kicked off with a stream reset and the read will fail + if err := r.ReadMsgWithContext(ctx, &claimAck); err != nil { + return nil, fmt.Errorf("read claim ack: %w", err) } - return nil + ch := make(chan []byte) + + // from now on we expect only to write updates to this stream + go func() { + defer stream.FullClose() + + for { + // we might want to do some input validation to see that the broker isn't tricking us + + select { + case v, ok := <-ch: + // closing the channel causes us to exit and close the stream + if !ok { + return + } + msg := pb.Broadcast{Soc: v} + if err := w.WriteMsgWithContext(ctx, &msg); err != nil { + s.logger.Error(err, "read join ack") + return + } + } + } + }() + return ch, nil } -func (s *Service) Publish(ctx context.Context, topic []byte, message []byte) error { - // grabs an existing publish stream or creates it then pushes the message down the pipe - // the stream stays alive and is reused later on for future messages - return nil -} +func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { + w, r := protobuf.NewWriterAndReader(stream) + defer stream.FullClose() -// handlerMessage handles incoming messages. on non-brokers, it tries to find the -// topic in the joined map, then notify the subscribed reader off the channel. -// the stream is reused and kept with a long-running goroutine. -func (s *Service) handlerMessage(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { - _, r := protobuf.NewWriterAndReader(stream) - //defer stream.FullClose() // NO FULL CLOSE HERE - // - var msg pb.PayloadMessage - if err := r.ReadMsgWithContext(ctx, &msg); err != nil { + var sysMsg pb.SystemMessage + if err := r.ReadMsgWithContext(ctx, &sysMsg); err != nil { return fmt.Errorf("read sys message: %w", err) } - if publish := msg.GetPublish(); publish != nil { - // we got a publish message - this means we are a broker. - // if there's a cohort - publish the message to the peers. launch a goroutine - // and read off the channel and feed it to the different readers + if join := sysMsg.GetJoin(); join != nil { + // peer is trying to join the cohort. accept and return the challenge + ack := pb.JoinAck{Challenge: []byte{0, 1, 2, 3}} + if err := w.WriteMsgWithContext(ctx, &ack); err != nil { + return fmt.Errorf("write claim: %w", err) + } return nil } - if bcast := msg.GetBroadcast(); bcast != nil { - // we got a broadcast on this stream, it means we are a subscriber here. - // find the subscriber channel(s), launch polling goroutine and duly notify - // when something comes in on the stream + if claim := sysMsg.GetClaim(); claim != nil { + // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off return nil } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index 5357816299c..be126e0d131 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -7,6 +7,7 @@ package bps_test import ( "bytes" "context" + "fmt" "testing" "github.com/ethersphere/bee/v2/pkg/bps" @@ -17,7 +18,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/swarm" ) -func TestGreet(t *testing.T) { +func TestJoin(t *testing.T) { t.Parallel() logger := log.Noop @@ -31,29 +32,27 @@ func TestGreet(t *testing.T) { client := bps.New(recorder, logger) addr := swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c") - greeting := "world" + topic := []byte{0, 1, 2, 3, 4} + // greeting := "world" - response, err := client.Greet(context.Background(), addr, greeting) + challenge, _, err := client.Join(context.Background(), addr, topic) if err != nil { t.Fatal(err) } - - if want := "hello, world"; response != want { - t.Fatalf("got response %q, want %q", response, want) - } - + fmt.Println(challenge) records, err := recorder.Records(addr, "bps", "1.0.0", "bps") if err != nil { t.Fatal(err) } if l := len(records); l != 1 { - t.Fatalf("got %v records, want %v", l, 1) + t.Fatalf("got %v records, want %v", l, 2) } record := records[0] + // client -> server: SystemMessage{Join} messages, err := protobuf.ReadMessages( bytes.NewReader(record.In()), - func() protobuf.Message { return new(pb.Hello) }, + func() protobuf.Message { return new(pb.SystemMessage) }, ) if err != nil { t.Fatal(err) @@ -61,13 +60,18 @@ func TestGreet(t *testing.T) { if l := len(messages); l != 1 { t.Fatalf("got %v messages, want %v", l, 1) } - if got := messages[0].(*pb.Hello).Greeting; got != greeting { - t.Fatalf("got greeting %q, want %q", got, greeting) + join := messages[0].(*pb.SystemMessage).GetJoin() + if join == nil { + t.Fatal("expected join message") + } + if !bytes.Equal(join.Topic, topic) { + t.Fatalf("got topic %x, want %x", join.Topic, topic) } + // server -> client returns challenge messages, err = protobuf.ReadMessages( bytes.NewReader(record.Out()), - func() protobuf.Message { return new(pb.Welcome) }, + func() protobuf.Message { return new(pb.JoinAck) }, ) if err != nil { t.Fatal(err) @@ -75,11 +79,11 @@ func TestGreet(t *testing.T) { if l := len(messages); l != 1 { t.Fatalf("got %v messages, want %v", l, 1) } - if got, want := messages[0].(*pb.Welcome).Response, "hello, world"; got != want { - t.Fatalf("got response %q, want %q", got, want) + cl := messages[0].(*pb.JoinAck).Challenge + if cl == nil { + t.Fatal("expected joinack message") } - - if err := record.Err(); err != nil { - t.Fatal(err) + if !bytes.Equal(challenge, cl) { + t.Fatalf("got challenge %x, want %x", cl, challenge) } } diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index a0d79961608..39407d31f29 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -71,135 +71,6 @@ func (m *Broadcast) GetSoc() []byte { return nil } -type Publish struct { - Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` -} - -func (m *Publish) Reset() { *m = Publish{} } -func (m *Publish) String() string { return proto.CompactTextString(m) } -func (*Publish) ProtoMessage() {} -func (*Publish) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{1} -} -func (m *Publish) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Publish) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Publish.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Publish) XXX_Merge(src proto.Message) { - xxx_messageInfo_Publish.Merge(m, src) -} -func (m *Publish) XXX_Size() int { - return m.Size() -} -func (m *Publish) XXX_DiscardUnknown() { - xxx_messageInfo_Publish.DiscardUnknown(m) -} - -var xxx_messageInfo_Publish proto.InternalMessageInfo - -func (m *Publish) GetSoc() []byte { - if m != nil { - return m.Soc - } - return nil -} - -type PayloadMessage struct { - // Types that are valid to be assigned to PayloadMessage: - // *PayloadMessage_Publish - // *PayloadMessage_Broadcast - PayloadMessage isPayloadMessage_PayloadMessage `protobuf_oneof:"payload_message"` -} - -func (m *PayloadMessage) Reset() { *m = PayloadMessage{} } -func (m *PayloadMessage) String() string { return proto.CompactTextString(m) } -func (*PayloadMessage) ProtoMessage() {} -func (*PayloadMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{2} -} -func (m *PayloadMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *PayloadMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_PayloadMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *PayloadMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_PayloadMessage.Merge(m, src) -} -func (m *PayloadMessage) XXX_Size() int { - return m.Size() -} -func (m *PayloadMessage) XXX_DiscardUnknown() { - xxx_messageInfo_PayloadMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_PayloadMessage proto.InternalMessageInfo - -type isPayloadMessage_PayloadMessage interface { - isPayloadMessage_PayloadMessage() - MarshalTo([]byte) (int, error) - Size() int -} - -type PayloadMessage_Publish struct { - Publish *Publish `protobuf:"bytes,1,opt,name=publish,proto3,oneof" json:"publish,omitempty"` -} -type PayloadMessage_Broadcast struct { - Broadcast *Broadcast `protobuf:"bytes,2,opt,name=broadcast,proto3,oneof" json:"broadcast,omitempty"` -} - -func (*PayloadMessage_Publish) isPayloadMessage_PayloadMessage() {} -func (*PayloadMessage_Broadcast) isPayloadMessage_PayloadMessage() {} - -func (m *PayloadMessage) GetPayloadMessage() isPayloadMessage_PayloadMessage { - if m != nil { - return m.PayloadMessage - } - return nil -} - -func (m *PayloadMessage) GetPublish() *Publish { - if x, ok := m.GetPayloadMessage().(*PayloadMessage_Publish); ok { - return x.Publish - } - return nil -} - -func (m *PayloadMessage) GetBroadcast() *Broadcast { - if x, ok := m.GetPayloadMessage().(*PayloadMessage_Broadcast); ok { - return x.Broadcast - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*PayloadMessage) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*PayloadMessage_Publish)(nil), - (*PayloadMessage_Broadcast)(nil), - } -} - type Join struct { Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` } @@ -208,7 +79,7 @@ func (m *Join) Reset() { *m = Join{} } func (m *Join) String() string { return proto.CompactTextString(m) } func (*Join) ProtoMessage() {} func (*Join) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{3} + return fileDescriptor_786299925f8760d5, []int{1} } func (m *Join) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -252,7 +123,7 @@ func (m *JoinAck) Reset() { *m = JoinAck{} } func (m *JoinAck) String() string { return proto.CompactTextString(m) } func (*JoinAck) ProtoMessage() {} func (*JoinAck) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{4} + return fileDescriptor_786299925f8760d5, []int{2} } func (m *JoinAck) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -289,14 +160,14 @@ func (m *JoinAck) GetChallenge() []byte { } type Claim struct { - ChallengeSig []byte `protobuf:"bytes,1,opt,name=challengeSig,proto3" json:"challengeSig,omitempty"` + Sig []byte `protobuf:"bytes,1,opt,name=sig,proto3" json:"sig,omitempty"` } func (m *Claim) Reset() { *m = Claim{} } func (m *Claim) String() string { return proto.CompactTextString(m) } func (*Claim) ProtoMessage() {} func (*Claim) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{5} + return fileDescriptor_786299925f8760d5, []int{3} } func (m *Claim) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -325,13 +196,49 @@ func (m *Claim) XXX_DiscardUnknown() { var xxx_messageInfo_Claim proto.InternalMessageInfo -func (m *Claim) GetChallengeSig() []byte { +func (m *Claim) GetSig() []byte { if m != nil { - return m.ChallengeSig + return m.Sig } return nil } +type ClaimAck struct { +} + +func (m *ClaimAck) Reset() { *m = ClaimAck{} } +func (m *ClaimAck) String() string { return proto.CompactTextString(m) } +func (*ClaimAck) ProtoMessage() {} +func (*ClaimAck) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{4} +} +func (m *ClaimAck) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ClaimAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ClaimAck.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ClaimAck) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClaimAck.Merge(m, src) +} +func (m *ClaimAck) XXX_Size() int { + return m.Size() +} +func (m *ClaimAck) XXX_DiscardUnknown() { + xxx_messageInfo_ClaimAck.DiscardUnknown(m) +} + +var xxx_messageInfo_ClaimAck proto.InternalMessageInfo + type SystemMessage struct { // Types that are valid to be assigned to SysMessage: // *SystemMessage_Join @@ -343,7 +250,7 @@ func (m *SystemMessage) Reset() { *m = SystemMessage{} } func (m *SystemMessage) String() string { return proto.CompactTextString(m) } func (*SystemMessage) ProtoMessage() {} func (*SystemMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{6} + return fileDescriptor_786299925f8760d5, []int{5} } func (m *SystemMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -419,38 +326,33 @@ func (*SystemMessage) XXX_OneofWrappers() []interface{} { func init() { proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") - proto.RegisterType((*Publish)(nil), "bps.Publish") - proto.RegisterType((*PayloadMessage)(nil), "bps.PayloadMessage") proto.RegisterType((*Join)(nil), "bps.Join") proto.RegisterType((*JoinAck)(nil), "bps.JoinAck") proto.RegisterType((*Claim)(nil), "bps.Claim") + proto.RegisterType((*ClaimAck)(nil), "bps.ClaimAck") proto.RegisterType((*SystemMessage)(nil), "bps.SystemMessage") } func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 311 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcb, 0x4a, 0xc3, 0x40, - 0x14, 0x86, 0x27, 0xbd, 0x58, 0x72, 0x7a, 0x51, 0x07, 0x17, 0x05, 0xeb, 0x28, 0xb3, 0xb1, 0x20, - 0x74, 0xa1, 0x4f, 0x60, 0xdc, 0x04, 0x41, 0x28, 0xe9, 0xce, 0x8d, 0xcc, 0x4c, 0x43, 0x3a, 0x9a, - 0x64, 0x86, 0x4e, 0x5c, 0x64, 0xe1, 0x3b, 0xf8, 0x58, 0x2e, 0xbb, 0x74, 0x29, 0xc9, 0x8b, 0x48, - 0x26, 0x49, 0x8b, 0xe0, 0x2e, 0x39, 0xff, 0xc7, 0xf9, 0xce, 0xcf, 0x80, 0xcb, 0xb5, 0x59, 0xe8, - 0xad, 0xca, 0x14, 0xee, 0x72, 0x6d, 0xe8, 0x05, 0xb8, 0xde, 0x56, 0xb1, 0xb5, 0x60, 0x26, 0xc3, - 0x27, 0xd0, 0x35, 0x4a, 0x4c, 0x9d, 0x2b, 0x67, 0x3e, 0x0a, 0xaa, 0x4f, 0x7a, 0x0e, 0x83, 0xe5, - 0x3b, 0x8f, 0xa5, 0xd9, 0xfc, 0x13, 0x7e, 0xc0, 0x64, 0xc9, 0xf2, 0x58, 0xb1, 0xf5, 0x53, 0x68, - 0x0c, 0x8b, 0x42, 0x3c, 0x87, 0x81, 0xae, 0x71, 0xcb, 0x0d, 0x6f, 0x47, 0x8b, 0xca, 0xd7, 0xac, - 0xf0, 0x51, 0xd0, 0xc6, 0x78, 0x01, 0x2e, 0x6f, 0xbd, 0xd3, 0x8e, 0x65, 0x27, 0x96, 0xdd, 0x5f, - 0xe3, 0xa3, 0xe0, 0x80, 0x78, 0xa7, 0x70, 0xac, 0x6b, 0xd7, 0x4b, 0x52, 0xcb, 0xe8, 0x0c, 0x7a, - 0x8f, 0x4a, 0xa6, 0xf8, 0x0c, 0xfa, 0x99, 0xd2, 0xb2, 0x3d, 0xad, 0xfe, 0xa1, 0xd7, 0x30, 0xa8, - 0xd2, 0x7b, 0xf1, 0x86, 0x67, 0xe0, 0x8a, 0x0d, 0x8b, 0xe3, 0x30, 0x8d, 0xc2, 0x06, 0x3a, 0x0c, - 0xe8, 0x0d, 0xf4, 0x1f, 0x62, 0x26, 0x13, 0x4c, 0x61, 0xb4, 0x9f, 0xae, 0x64, 0xd4, 0x90, 0x7f, - 0x66, 0x54, 0xc0, 0x78, 0x95, 0x9b, 0x2c, 0x4c, 0xda, 0xc6, 0x97, 0xd0, 0x7b, 0x55, 0x32, 0x6d, - 0xea, 0xba, 0xb6, 0x42, 0xe5, 0xf5, 0x51, 0x60, 0x03, 0x4c, 0xa1, 0x2f, 0xaa, 0xf5, 0x4d, 0x49, - 0xb0, 0x84, 0x15, 0xfa, 0x28, 0xa8, 0x23, 0x6f, 0x0c, 0x43, 0x93, 0x9b, 0xb6, 0x98, 0x37, 0xfb, - 0x2a, 0x88, 0xb3, 0x2b, 0x88, 0xf3, 0x53, 0x10, 0xe7, 0xb3, 0x24, 0x68, 0x57, 0x12, 0xf4, 0x5d, - 0x12, 0xf4, 0xdc, 0xd1, 0x9c, 0x1f, 0xd9, 0xd7, 0xbb, 0xfb, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x59, - 0x6f, 0x46, 0x7c, 0xca, 0x01, 0x00, 0x00, + // 249 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x90, 0xbd, 0x4e, 0xc3, 0x30, + 0x14, 0x85, 0xed, 0xb6, 0x01, 0x72, 0x4b, 0x25, 0x64, 0x31, 0x14, 0x29, 0x18, 0xe4, 0x05, 0xa6, + 0x0e, 0xf0, 0x04, 0x0d, 0x4b, 0x85, 0xc4, 0x12, 0x36, 0x16, 0xe4, 0x18, 0x2b, 0x98, 0x26, 0xb1, + 0xd5, 0xeb, 0xa5, 0x6f, 0xc1, 0x63, 0x31, 0x66, 0x64, 0x44, 0xc9, 0x8b, 0x20, 0x3b, 0xfc, 0x6c, + 0xb6, 0xbf, 0xcf, 0xe7, 0x5c, 0x5d, 0x48, 0x4b, 0x87, 0x2b, 0xb7, 0xb3, 0xde, 0xb2, 0x69, 0xe9, + 0x50, 0x9c, 0x43, 0x9a, 0xef, 0xac, 0x7c, 0x51, 0x12, 0x3d, 0x3b, 0x81, 0x29, 0x5a, 0xb5, 0xa4, + 0x97, 0xf4, 0xfa, 0xb8, 0x08, 0x47, 0x91, 0xc1, 0xec, 0xde, 0x9a, 0x96, 0x9d, 0x42, 0xe2, 0xad, + 0x33, 0xbf, 0x6c, 0xbc, 0x88, 0x2b, 0x38, 0x0c, 0x74, 0xad, 0xb6, 0x2c, 0x83, 0x54, 0xbd, 0xca, + 0xba, 0xd6, 0x6d, 0xa5, 0x7f, 0xa4, 0xff, 0x07, 0x71, 0x06, 0xc9, 0x5d, 0x2d, 0x4d, 0x13, 0x1b, + 0x4c, 0xf5, 0xd7, 0x60, 0x2a, 0x01, 0x70, 0x14, 0xd1, 0x5a, 0x6d, 0x85, 0x82, 0xc5, 0xe3, 0x1e, + 0xbd, 0x6e, 0x1e, 0x34, 0xa2, 0xac, 0x34, 0xbb, 0x80, 0xd9, 0x9b, 0x35, 0x6d, 0xf4, 0xe7, 0x37, + 0xe9, 0x2a, 0x0c, 0x1f, 0x1a, 0x37, 0xa4, 0x88, 0x80, 0x09, 0x48, 0x54, 0xf8, 0xbd, 0x9c, 0x44, + 0x03, 0xa2, 0x11, 0xf3, 0x36, 0xa4, 0x18, 0x51, 0xbe, 0x80, 0x39, 0xee, 0xf1, 0xb9, 0x19, 0x33, + 0xf3, 0xec, 0xa3, 0xe7, 0xb4, 0xeb, 0x39, 0xfd, 0xea, 0x39, 0x7d, 0x1f, 0x38, 0xe9, 0x06, 0x4e, + 0x3e, 0x07, 0x4e, 0x9e, 0x26, 0xae, 0x2c, 0x0f, 0xe2, 0x6e, 0x6e, 0xbf, 0x03, 0x00, 0x00, 0xff, + 0xff, 0x8a, 0xa8, 0x0a, 0x8d, 0x28, 0x01, 0x00, 0x00, } func (m *Broadcast) Marshal() (dAtA []byte, err error) { @@ -483,37 +385,7 @@ func (m *Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *Publish) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Publish) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Publish) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Soc) > 0 { - i -= len(m.Soc) - copy(dAtA[i:], m.Soc) - i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *PayloadMessage) Marshal() (dAtA []byte, err error) { +func (m *Join) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -523,71 +395,27 @@ func (m *PayloadMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *PayloadMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *Join) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *PayloadMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.PayloadMessage != nil { - { - size := m.PayloadMessage.Size() - i -= size - if _, err := m.PayloadMessage.MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - } - } - return len(dAtA) - i, nil -} - -func (m *PayloadMessage_Publish) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *PayloadMessage_Publish) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - if m.Publish != nil { - { - size, err := m.Publish.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintBps(dAtA, i, uint64(size)) - } + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } -func (m *PayloadMessage_Broadcast) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} -func (m *PayloadMessage_Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - if m.Broadcast != nil { - { - size, err := m.Broadcast.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintBps(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - return len(dAtA) - i, nil -} -func (m *Join) Marshal() (dAtA []byte, err error) { +func (m *JoinAck) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -597,27 +425,27 @@ func (m *Join) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Join) MarshalTo(dAtA []byte) (int, error) { +func (m *JoinAck) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *JoinAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.Topic) > 0 { - i -= len(m.Topic) - copy(dAtA[i:], m.Topic) - i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) + if len(m.Challenge) > 0 { + i -= len(m.Challenge) + copy(dAtA[i:], m.Challenge) + i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } -func (m *JoinAck) Marshal() (dAtA []byte, err error) { +func (m *Claim) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -627,27 +455,27 @@ func (m *JoinAck) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *JoinAck) MarshalTo(dAtA []byte) (int, error) { +func (m *Claim) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *JoinAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.Challenge) > 0 { - i -= len(m.Challenge) - copy(dAtA[i:], m.Challenge) - i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) + if len(m.Sig) > 0 { + i -= len(m.Sig) + copy(dAtA[i:], m.Sig) + i = encodeVarintBps(dAtA, i, uint64(len(m.Sig))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } -func (m *Claim) Marshal() (dAtA []byte, err error) { +func (m *ClaimAck) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -657,23 +485,16 @@ func (m *Claim) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Claim) MarshalTo(dAtA []byte) (int, error) { +func (m *ClaimAck) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *ClaimAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.ChallengeSig) > 0 { - i -= len(m.ChallengeSig) - copy(dAtA[i:], m.ChallengeSig) - i = encodeVarintBps(dAtA, i, uint64(len(m.ChallengeSig))) - i-- - dAtA[i] = 0xa - } return len(dAtA) - i, nil } @@ -775,91 +596,51 @@ func (m *Broadcast) Size() (n int) { return n } -func (m *Publish) Size() (n int) { +func (m *Join) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Soc) + l = len(m.Topic) if l > 0 { n += 1 + l + sovBps(uint64(l)) } return n } -func (m *PayloadMessage) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.PayloadMessage != nil { - n += m.PayloadMessage.Size() - } - return n -} - -func (m *PayloadMessage_Publish) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Publish != nil { - l = m.Publish.Size() - n += 1 + l + sovBps(uint64(l)) - } - return n -} -func (m *PayloadMessage_Broadcast) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Broadcast != nil { - l = m.Broadcast.Size() - n += 1 + l + sovBps(uint64(l)) - } - return n -} -func (m *Join) Size() (n int) { +func (m *JoinAck) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Topic) + l = len(m.Challenge) if l > 0 { n += 1 + l + sovBps(uint64(l)) } return n } -func (m *JoinAck) Size() (n int) { +func (m *Claim) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Challenge) + l = len(m.Sig) if l > 0 { n += 1 + l + sovBps(uint64(l)) } return n } -func (m *Claim) Size() (n int) { +func (m *ClaimAck) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.ChallengeSig) - if l > 0 { - n += 1 + l + sovBps(uint64(l)) - } return n } @@ -993,7 +774,7 @@ func (m *Broadcast) Unmarshal(dAtA []byte) error { } return nil } -func (m *Publish) Unmarshal(dAtA []byte) error { +func (m *Join) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1016,15 +797,15 @@ func (m *Publish) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Publish: wiretype end group for non-group") + return fmt.Errorf("proto: Join: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Publish: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Join: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1051,133 +832,10 @@ func (m *Publish) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) - if m.Soc == nil { - m.Soc = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipBps(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *PayloadMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: PayloadMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: PayloadMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Publish", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - v := &Publish{} - if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - m.PayloadMessage = &PayloadMessage_Publish{v} - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Broadcast", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - v := &Broadcast{} - if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err + m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) + if m.Topic == nil { + m.Topic = []byte{} } - m.PayloadMessage = &PayloadMessage_Broadcast{v} iNdEx = postIndex default: iNdEx = preIndex @@ -1203,7 +861,7 @@ func (m *PayloadMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *Join) Unmarshal(dAtA []byte) error { +func (m *JoinAck) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1226,15 +884,15 @@ func (m *Join) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Join: wiretype end group for non-group") + return fmt.Errorf("proto: JoinAck: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Join: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: JoinAck: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1261,9 +919,9 @@ func (m *Join) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) - if m.Topic == nil { - m.Topic = []byte{} + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} } iNdEx = postIndex default: @@ -1290,7 +948,7 @@ func (m *Join) Unmarshal(dAtA []byte) error { } return nil } -func (m *JoinAck) Unmarshal(dAtA []byte) error { +func (m *Claim) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1313,15 +971,15 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: JoinAck: wiretype end group for non-group") + return fmt.Errorf("proto: Claim: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: JoinAck: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Claim: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Sig", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1348,9 +1006,9 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) - if m.Challenge == nil { - m.Challenge = []byte{} + m.Sig = append(m.Sig[:0], dAtA[iNdEx:postIndex]...) + if m.Sig == nil { + m.Sig = []byte{} } iNdEx = postIndex default: @@ -1377,7 +1035,7 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { } return nil } -func (m *Claim) Unmarshal(dAtA []byte) error { +func (m *ClaimAck) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1400,46 +1058,12 @@ func (m *Claim) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Claim: wiretype end group for non-group") + return fmt.Errorf("proto: ClaimAck: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Claim: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ClaimAck: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ChallengeSig", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ChallengeSig = append(m.ChallengeSig[:0], dAtA[iNdEx:postIndex]...) - if m.ChallengeSig == nil { - m.ChallengeSig = []byte{} - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipBps(dAtA[iNdEx:]) diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index 243d42846cb..a2fbb396c2b 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -20,17 +20,6 @@ message Broadcast { bytes soc = 1; } -message Publish { - bytes soc = 1; -} - -message PayloadMessage { - oneof payload_message { - Publish publish = 1; - Broadcast broadcast = 2; - } -} - message Join { bytes topic = 1; } @@ -40,9 +29,11 @@ message JoinAck { } message Claim { - bytes challengeSig = 1; // the challenge signature that takes over the cohort. signs the challenge in JoinAck. + bytes sig = 1; // the challenge signature that takes over the cohort. signs the challenge in JoinAck. } +message ClaimAck {} + message SystemMessage { oneof sys_message { From d69711f7ed9445c6ab9742d47597b3a0d5928ef4 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:16:58 -0600 Subject: [PATCH 04/29] chore: general outline (cherry picked from commit 700a56be0ff4ee3b51639efddd982c8cccff5938) --- pkg/bps/bps.go | 61 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index ee80a566e90..d293f9758a1 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -27,12 +27,6 @@ const ( streamName = "bps" ) -// Interface is the main interface of the bps protocol. -//type Interface interface { -//Join(ctx context.Context, p p2p.Peer, topic []byte, notify <-chan []byte) error -//Publish(ctx context.Context, topic []byte, message []byte) error -//} - // Service is the bps protocol service. type Service struct { mtx sync.Mutex @@ -66,9 +60,8 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } } -// Join onto a topic at the remote peer. The channel passed at registration will be notified -// with the payloads once they arrive. Returns the challenge and a channel that would send -// the payloads over it. +// Join onto a topic at the broker at address. +// Returns the challenge and a channel that would send the payloads over it. func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) ([]byte, chan []byte, error) { stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) if err != nil { @@ -112,8 +105,8 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) return joinAck.Challenge, rxCh, nil } -// claim a given topic on a broker with the provided signature. returns the write channel used in order -// to send later payloads +// claim a given topic on a broker (at address) with the provided signature. +// returns the write channel used in order to send later payloads. func (s *Service) Claim(ctx context.Context, address swarm.Address, topic, sig []byte) (chan []byte, error) { stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) if err != nil { @@ -161,10 +154,10 @@ func (s *Service) Claim(ctx context.Context, address swarm.Address, topic, sig [ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { w, r := protobuf.NewWriterAndReader(stream) - defer stream.FullClose() var sysMsg pb.SystemMessage if err := r.ReadMsgWithContext(ctx, &sysMsg); err != nil { + go stream.FullClose() return fmt.Errorf("read sys message: %w", err) } @@ -172,13 +165,55 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er // peer is trying to join the cohort. accept and return the challenge ack := pb.JoinAck{Challenge: []byte{0, 1, 2, 3}} if err := w.WriteMsgWithContext(ctx, &ack); err != nil { + go stream.FullClose() return fmt.Errorf("write claim: %w", err) } + + // add to the cohort and get a channel to receive the broadcasts on + ch := make(chan []byte) // replace with the registry channel later + go func() { + defer stream.FullClose() + for { + // await messages, then write to the stream once they come in + select { + case msg := <-ch: + m := pb.Broadcast{Soc: msg} + if err := w.WriteMsgWithContext(ctx, &m); err != nil { + s.logger.Error(err, "write broadcast") + return + } + case <-ctx.Done(): + return + } + } + }() return nil } if claim := sysMsg.GetClaim(); claim != nil { // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off - + //_ = claim.Challenge + // check if claim signature pk matches the feed owner address on the registry + // if it does - this is an atomic swap - the current stream becomes the publisher stream + // and the next challenge changes randomly, so that if needed, the publisher can reclaim + // later and rejoin + reclaim. + + ch := make(chan []byte) // replace with the cohort send channel later + go func() { + defer stream.FullClose() + for { + // await messages, then write to the cohort channel + m := pb.Broadcast{} + if err := r.ReadMsgWithContext(ctx, &m); err != nil { + s.logger.Error(err, "read broadcast") + return + } + select { + case ch <- m.Soc: + case <-ctx.Done(): + return + } + } + }() return nil } From 1e758efa0b0a60783cfe5bdc86e16cb546858950 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:22:52 -0600 Subject: [PATCH 05/29] chore: small things (cherry picked from commit 81a5d96f440147d988034580f915cf7380a016cf) --- pkg/bps/bps.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index d293f9758a1..2d60b5da40d 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -8,6 +8,7 @@ package bps import ( "context" + "errors" "fmt" "sync" @@ -27,11 +28,16 @@ const ( streamName = "bps" ) +var errNotBroker = errors.New("not a broker") + // Service is the bps protocol service. type Service struct { mtx sync.Mutex + fullNode bool streamer p2p.Streamer + // cohort registry - keep track of members, publisher, channels, challenge, streams? + // this is only relevant for the actual broker. subscribers don't care of manage this state at all. registry *CohortRegistry logger log.Logger @@ -60,7 +66,7 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } } -// Join onto a topic at the broker at address. +// Join onto a topic at the broker at address. This is called on the subscriber. // Returns the challenge and a channel that would send the payloads over it. func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) ([]byte, chan []byte, error) { stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) @@ -106,6 +112,7 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) } // claim a given topic on a broker (at address) with the provided signature. +// this is called on the subsciber that wants to become a publisher. // returns the write channel used in order to send later payloads. func (s *Service) Claim(ctx context.Context, address swarm.Address, topic, sig []byte) (chan []byte, error) { stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) @@ -152,7 +159,12 @@ func (s *Service) Claim(ctx context.Context, address swarm.Address, topic, sig [ return ch, nil } +// handler is the protocol handler on the broker. func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { + if !s.fullNode { + stream.Reset() + return errNotBroker + } w, r := protobuf.NewWriterAndReader(stream) var sysMsg pb.SystemMessage From dde6d43951907ff5f7b20a8f5ff2d0cab4526fac Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Thu, 24 Sep 2026 09:18:47 -0600 Subject: [PATCH 06/29] chore: simplify further (cherry picked from commit d28ced35d96fe56c9d589aa6b3276ecdff59a178) --- pkg/bps/bps.go | 115 ++++++----- pkg/bps/pb/bps.pb.go | 471 ++----------------------------------------- pkg/bps/pb/bps.proto | 12 +- 3 files changed, 75 insertions(+), 523 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 2d60b5da40d..5b51e1ee19b 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -167,67 +167,76 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } w, r := protobuf.NewWriterAndReader(stream) - var sysMsg pb.SystemMessage - if err := r.ReadMsgWithContext(ctx, &sysMsg); err != nil { + var join pb.Join + if err := r.ReadMsgWithContext(ctx, &join); err != nil { go stream.FullClose() return fmt.Errorf("read sys message: %w", err) } - if join := sysMsg.GetJoin(); join != nil { - // peer is trying to join the cohort. accept and return the challenge - ack := pb.JoinAck{Challenge: []byte{0, 1, 2, 3}} - if err := w.WriteMsgWithContext(ctx, &ack); err != nil { - go stream.FullClose() - return fmt.Errorf("write claim: %w", err) - } - - // add to the cohort and get a channel to receive the broadcasts on - ch := make(chan []byte) // replace with the registry channel later - go func() { - defer stream.FullClose() - for { - // await messages, then write to the stream once they come in - select { - case msg := <-ch: - m := pb.Broadcast{Soc: msg} - if err := w.WriteMsgWithContext(ctx, &m); err != nil { - s.logger.Error(err, "write broadcast") - return - } - case <-ctx.Done(): - return - } - } - }() - return nil + // peer is trying to join the cohort. accept and return the challenge + ack := pb.JoinAck{Challenge: []byte{0, 1, 2, 3}} + if err := w.WriteMsgWithContext(ctx, &ack); err != nil { + go stream.FullClose() + return fmt.Errorf("write claim: %w", err) } - if claim := sysMsg.GetClaim(); claim != nil { - // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off - //_ = claim.Challenge - // check if claim signature pk matches the feed owner address on the registry - // if it does - this is an atomic swap - the current stream becomes the publisher stream - // and the next challenge changes randomly, so that if needed, the publisher can reclaim - // later and rejoin + reclaim. - - ch := make(chan []byte) // replace with the cohort send channel later - go func() { - defer stream.FullClose() - for { - // await messages, then write to the cohort channel - m := pb.Broadcast{} - if err := r.ReadMsgWithContext(ctx, &m); err != nil { - s.logger.Error(err, "read broadcast") - return - } - select { - case ch <- m.Soc: - case <-ctx.Done(): + + // add to the cohort and get a channel to receive the broadcasts on + ch := make(chan []byte) // replace with the registry channel later + + // the writer side - reads messages off the publisher channel + // and pushes them to the subscriber stream + go func() { + defer stream.FullClose() + for { + // await messages, then write to the stream once they come in + select { + case msg := <-ch: + m := pb.Broadcast{Soc: msg} + if err := w.WriteMsgWithContext(ctx, &m); err != nil { + s.logger.Error(err, "write broadcast") return } + case <-ctx.Done(): + return } - }() - return nil - } + } + }() + + // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off + //_ = claim.Challenge + // check if claim signature pk matches the feed owner address on the registry + // if it does - this is an atomic swap - the current stream becomes the publisher stream + // and the next challenge changes randomly, so that if needed, the publisher can reclaim + // later and rejoin + reclaim. + + // writer side - we first try to read a claim. we can wait indefinitely here. + // once a claim is accepted, we prompte the stream to be a publisher stream + // then continuously try to read from it broadcast messages and push them over the + // publisher channel + ch1 := make(chan []byte) // replace with the cohort send channel later + go func() { + defer stream.FullClose() + claim := pb.Claim{} + if err := r.ReadMsgWithContext(ctx, &claim); err != nil { + s.logger.Error(err, "read claim") + return + } + // do the checks on the claim, if it is wrong then we reset the stream + + for { + // await messages, then write to the cohort channel + m := pb.Broadcast{} + if err := r.ReadMsgWithContext(ctx, &m); err != nil { + s.logger.Error(err, "read broadcast") + return + } + select { + case ch1 <- m.Soc: + case <-ctx.Done(): + return + } + } + }() return nil } diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index 39407d31f29..35a5d69fb34 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -203,156 +203,28 @@ func (m *Claim) GetSig() []byte { return nil } -type ClaimAck struct { -} - -func (m *ClaimAck) Reset() { *m = ClaimAck{} } -func (m *ClaimAck) String() string { return proto.CompactTextString(m) } -func (*ClaimAck) ProtoMessage() {} -func (*ClaimAck) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{4} -} -func (m *ClaimAck) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ClaimAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ClaimAck.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ClaimAck) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClaimAck.Merge(m, src) -} -func (m *ClaimAck) XXX_Size() int { - return m.Size() -} -func (m *ClaimAck) XXX_DiscardUnknown() { - xxx_messageInfo_ClaimAck.DiscardUnknown(m) -} - -var xxx_messageInfo_ClaimAck proto.InternalMessageInfo - -type SystemMessage struct { - // Types that are valid to be assigned to SysMessage: - // *SystemMessage_Join - // *SystemMessage_Claim - SysMessage isSystemMessage_SysMessage `protobuf_oneof:"sys_message"` -} - -func (m *SystemMessage) Reset() { *m = SystemMessage{} } -func (m *SystemMessage) String() string { return proto.CompactTextString(m) } -func (*SystemMessage) ProtoMessage() {} -func (*SystemMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{5} -} -func (m *SystemMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SystemMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SystemMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *SystemMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_SystemMessage.Merge(m, src) -} -func (m *SystemMessage) XXX_Size() int { - return m.Size() -} -func (m *SystemMessage) XXX_DiscardUnknown() { - xxx_messageInfo_SystemMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_SystemMessage proto.InternalMessageInfo - -type isSystemMessage_SysMessage interface { - isSystemMessage_SysMessage() - MarshalTo([]byte) (int, error) - Size() int -} - -type SystemMessage_Join struct { - Join *Join `protobuf:"bytes,1,opt,name=join,proto3,oneof" json:"join,omitempty"` -} -type SystemMessage_Claim struct { - Claim *Claim `protobuf:"bytes,2,opt,name=claim,proto3,oneof" json:"claim,omitempty"` -} - -func (*SystemMessage_Join) isSystemMessage_SysMessage() {} -func (*SystemMessage_Claim) isSystemMessage_SysMessage() {} - -func (m *SystemMessage) GetSysMessage() isSystemMessage_SysMessage { - if m != nil { - return m.SysMessage - } - return nil -} - -func (m *SystemMessage) GetJoin() *Join { - if x, ok := m.GetSysMessage().(*SystemMessage_Join); ok { - return x.Join - } - return nil -} - -func (m *SystemMessage) GetClaim() *Claim { - if x, ok := m.GetSysMessage().(*SystemMessage_Claim); ok { - return x.Claim - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*SystemMessage) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*SystemMessage_Join)(nil), - (*SystemMessage_Claim)(nil), - } -} - func init() { proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") proto.RegisterType((*Join)(nil), "bps.Join") proto.RegisterType((*JoinAck)(nil), "bps.JoinAck") proto.RegisterType((*Claim)(nil), "bps.Claim") - proto.RegisterType((*ClaimAck)(nil), "bps.ClaimAck") - proto.RegisterType((*SystemMessage)(nil), "bps.SystemMessage") } func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 249 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x90, 0xbd, 0x4e, 0xc3, 0x30, - 0x14, 0x85, 0xed, 0xb6, 0x01, 0x72, 0x4b, 0x25, 0x64, 0x31, 0x14, 0x29, 0x18, 0xe4, 0x05, 0xa6, - 0x0e, 0xf0, 0x04, 0x0d, 0x4b, 0x85, 0xc4, 0x12, 0x36, 0x16, 0xe4, 0x18, 0x2b, 0x98, 0x26, 0xb1, - 0xd5, 0xeb, 0xa5, 0x6f, 0xc1, 0x63, 0x31, 0x66, 0x64, 0x44, 0xc9, 0x8b, 0x20, 0x3b, 0xfc, 0x6c, - 0xb6, 0xbf, 0xcf, 0xe7, 0x5c, 0x5d, 0x48, 0x4b, 0x87, 0x2b, 0xb7, 0xb3, 0xde, 0xb2, 0x69, 0xe9, - 0x50, 0x9c, 0x43, 0x9a, 0xef, 0xac, 0x7c, 0x51, 0x12, 0x3d, 0x3b, 0x81, 0x29, 0x5a, 0xb5, 0xa4, - 0x97, 0xf4, 0xfa, 0xb8, 0x08, 0x47, 0x91, 0xc1, 0xec, 0xde, 0x9a, 0x96, 0x9d, 0x42, 0xe2, 0xad, - 0x33, 0xbf, 0x6c, 0xbc, 0x88, 0x2b, 0x38, 0x0c, 0x74, 0xad, 0xb6, 0x2c, 0x83, 0x54, 0xbd, 0xca, - 0xba, 0xd6, 0x6d, 0xa5, 0x7f, 0xa4, 0xff, 0x07, 0x71, 0x06, 0xc9, 0x5d, 0x2d, 0x4d, 0x13, 0x1b, - 0x4c, 0xf5, 0xd7, 0x60, 0x2a, 0x01, 0x70, 0x14, 0xd1, 0x5a, 0x6d, 0x85, 0x82, 0xc5, 0xe3, 0x1e, - 0xbd, 0x6e, 0x1e, 0x34, 0xa2, 0xac, 0x34, 0xbb, 0x80, 0xd9, 0x9b, 0x35, 0x6d, 0xf4, 0xe7, 0x37, - 0xe9, 0x2a, 0x0c, 0x1f, 0x1a, 0x37, 0xa4, 0x88, 0x80, 0x09, 0x48, 0x54, 0xf8, 0xbd, 0x9c, 0x44, - 0x03, 0xa2, 0x11, 0xf3, 0x36, 0xa4, 0x18, 0x51, 0xbe, 0x80, 0x39, 0xee, 0xf1, 0xb9, 0x19, 0x33, - 0xf3, 0xec, 0xa3, 0xe7, 0xb4, 0xeb, 0x39, 0xfd, 0xea, 0x39, 0x7d, 0x1f, 0x38, 0xe9, 0x06, 0x4e, - 0x3e, 0x07, 0x4e, 0x9e, 0x26, 0xae, 0x2c, 0x0f, 0xe2, 0x6e, 0x6e, 0xbf, 0x03, 0x00, 0x00, 0xff, - 0xff, 0x8a, 0xa8, 0x0a, 0x8d, 0x28, 0x01, 0x00, 0x00, + // 168 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0x2a, 0x28, 0xd6, + 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x4e, 0x2a, 0x28, 0x56, 0x92, 0xe5, 0xe2, 0x74, 0x2a, + 0xca, 0x4f, 0x4c, 0x49, 0x4e, 0x2c, 0x2e, 0x11, 0x12, 0xe0, 0x62, 0x2e, 0xce, 0x4f, 0x96, 0x60, + 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0x31, 0x95, 0x64, 0xb8, 0x58, 0xbc, 0xf2, 0x33, 0xf3, 0x84, + 0x44, 0xb8, 0x58, 0x4b, 0xf2, 0x0b, 0x32, 0x61, 0x72, 0x10, 0x8e, 0x92, 0x3a, 0x17, 0x3b, 0x48, + 0xd6, 0x31, 0x39, 0x5b, 0x48, 0x86, 0x8b, 0x33, 0x39, 0x23, 0x31, 0x27, 0x27, 0x35, 0x2f, 0x3d, + 0x15, 0xaa, 0x08, 0x21, 0xa0, 0x24, 0xc9, 0xc5, 0xea, 0x9c, 0x93, 0x98, 0x99, 0x0b, 0xb6, 0x21, + 0x33, 0x1d, 0x6e, 0x43, 0x66, 0xba, 0x93, 0xcc, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, + 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, + 0x31, 0x44, 0x31, 0x15, 0x24, 0x25, 0xb1, 0x81, 0x9d, 0x6a, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, + 0x75, 0xae, 0x61, 0xc4, 0xb7, 0x00, 0x00, 0x00, } func (m *Broadcast) Marshal() (dAtA []byte, err error) { @@ -475,103 +347,6 @@ func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *ClaimAck) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ClaimAck) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ClaimAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - return len(dAtA) - i, nil -} - -func (m *SystemMessage) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *SystemMessage) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SystemMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.SysMessage != nil { - { - size := m.SysMessage.Size() - i -= size - if _, err := m.SysMessage.MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - } - } - return len(dAtA) - i, nil -} - -func (m *SystemMessage_Join) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SystemMessage_Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - if m.Join != nil { - { - size, err := m.Join.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintBps(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} -func (m *SystemMessage_Claim) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SystemMessage_Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - if m.Claim != nil { - { - size, err := m.Claim.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintBps(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - return len(dAtA) - i, nil -} func encodeVarintBps(dAtA []byte, offset int, v uint64) int { offset -= sovBps(v) base := offset @@ -635,52 +410,6 @@ func (m *Claim) Size() (n int) { return n } -func (m *ClaimAck) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - return n -} - -func (m *SystemMessage) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.SysMessage != nil { - n += m.SysMessage.Size() - } - return n -} - -func (m *SystemMessage_Join) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Join != nil { - l = m.Join.Size() - n += 1 + l + sovBps(uint64(l)) - } - return n -} -func (m *SystemMessage_Claim) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Claim != nil { - l = m.Claim.Size() - n += 1 + l + sovBps(uint64(l)) - } - return n -} - func sovBps(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -1035,182 +764,6 @@ func (m *Claim) Unmarshal(dAtA []byte) error { } return nil } -func (m *ClaimAck) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ClaimAck: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ClaimAck: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipBps(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *SystemMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SystemMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SystemMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Join", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - v := &Join{} - if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - m.SysMessage = &SystemMessage_Join{v} - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Claim", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthBps - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthBps - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - v := &Claim{} - if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - m.SysMessage = &SystemMessage_Claim{v} - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipBps(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthBps - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func skipBps(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index a2fbb396c2b..2123680675d 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -32,14 +32,4 @@ message Claim { bytes sig = 1; // the challenge signature that takes over the cohort. signs the challenge in JoinAck. } -message ClaimAck {} - - -message SystemMessage { - oneof sys_message { - Join join = 1; - Claim claim =2; - } -} - - +//message ClaimAck {} From 85e8170cacb330dd254923ae49710ae6fc978903 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:29:06 -0600 Subject: [PATCH 07/29] chore: cleanups (cherry picked from commit 94a38c9fb43dfd61f9d0575aac99644e25fc37b7) --- pkg/bps/pb/bps.proto | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index 2123680675d..0efa47b1365 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -8,14 +8,6 @@ package bps; option go_package = "pb"; - - - -// Both directions after the handshake: admin -> broker is a publication, broker -> -// subscriber a delivery of the same bytes. The single-owner chunk travels as its -// stored chunk data, opaque to the protocol and validated by the ordinary SOC code -// once the `id` slot has been rewritten as the Frames section says: -// id (32) || signature (65) || span (8, LE) || payload (<= 4096) message Broadcast { bytes soc = 1; } @@ -31,5 +23,3 @@ message JoinAck { message Claim { bytes sig = 1; // the challenge signature that takes over the cohort. signs the challenge in JoinAck. } - -//message ClaimAck {} From 5d5cf0e968429c56185ef977a85073945c57dfc0 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:06:23 -0600 Subject: [PATCH 08/29] refactor(bps): protocol registry --- pkg/bps/bps.go | 16 ++++------------ pkg/bps/registry.go | 41 ++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 5b51e1ee19b..ad9301732e9 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -75,9 +75,7 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) } w, r := protobuf.NewWriterAndReader(stream) - joinMsg := pb.SystemMessage{SysMessage: &pb.SystemMessage_Join{ - Join: &pb.Join{Topic: topic}, - }} + joinMsg := pb.Join{Topic: topic} if err := w.WriteMsgWithContext(ctx, &joinMsg); err != nil { return nil, nil, fmt.Errorf("write join msg: %w", err) } @@ -120,19 +118,12 @@ func (s *Service) Claim(ctx context.Context, address swarm.Address, topic, sig [ return nil, fmt.Errorf("new stream: %w", err) } - w, r := protobuf.NewWriterAndReader(stream) - claim := pb.SystemMessage{SysMessage: &pb.SystemMessage_Claim{ - Claim: &pb.Claim{Sig: topic}, - }} + w, _ := protobuf.NewWriterAndReader(stream) + claim := pb.Claim{Sig: topic} if err := w.WriteMsgWithContext(ctx, &claim); err != nil { return nil, fmt.Errorf("write claim: %w", err) } - claimAck := pb.ClaimAck{} // if the claim is wrong we will just get kicked off with a stream reset and the read will fail - if err := r.ReadMsgWithContext(ctx, &claimAck); err != nil { - return nil, fmt.Errorf("read claim ack: %w", err) - } - ch := make(chan []byte) // from now on we expect only to write updates to this stream @@ -173,6 +164,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er return fmt.Errorf("read sys message: %w", err) } + // register in the cohort and return the secret // peer is trying to join the cohort. accept and return the challenge ack := pb.JoinAck{Challenge: []byte{0, 1, 2, 3}} if err := w.WriteMsgWithContext(ctx, &ack); err != nil { diff --git a/pkg/bps/registry.go b/pkg/bps/registry.go index bbf96642fa9..20be39707be 100644 --- a/pkg/bps/registry.go +++ b/pkg/bps/registry.go @@ -1,6 +1,7 @@ package bps import ( + "crypto/rand" "sync" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -8,26 +9,48 @@ import ( func NewRegistry() *CohortRegistry { return &CohortRegistry{ - cohorts: make(map[string]*Cohort), + cohorts: make(map[string]*cohort), } } type CohortRegistry struct { mtx sync.Mutex - cohorts map[string]*Cohort + cohorts map[string]*cohort } -// Jopen is a join/open operation in one - the peer either joins an existing -// cohort or creates one by joining a previously unknown topic. -func (c *CohortRegistry) Jopen(overlay swarm.Address, topic []byte) *Cohort { - return nil +// join/open operation in one - the peer either joins an existing +// cohort or creates one by joining a previously unregistered topic. +// returns the cohort challenge +func (c *CohortRegistry) Join(overlay swarm.Address, topic []byte, ch chan []byte) []byte { + c.mtx.Lock() + defer c.mtx.Unlock() + t := string(topic) + co, ok := c.cohorts[t] + if ok { + co.members[overlay.String()] = ch + return co.challenge + } + members := make(map[string]chan []byte) + challenge := make([]byte, 32) + if _, err := rand.Read(challenge); err != nil { + panic(err) + } + + members[overlay.String()] = ch + c.cohorts[t] = &cohort{ + topic: topic, + members: members, + challenge: challenge, + } + + return challenge } -type Cohort struct { - mtx sync.Mutex +type cohort struct { + // mtx sync.Mutex topic []byte // topic is the feed topic + owner hash lastSeen uint64 // last feed update index, used to prevent replay and circumvent dedup logic (for now) - members map[string]string + members map[string]chan []byte publisher swarm.Address challenge []byte } From a8967c56c669817f25c62b46d7d1f70d3a5afb80 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:25:59 -0600 Subject: [PATCH 09/29] chore: condense, simplify (cherry picked from commit d06b3768225cce7d1b0522c042b1bf2d7c6c5388) --- pkg/bps/bps.go | 215 ++++++++++++++++++++++++++++++--------- pkg/bps/pb/bps.pb.go | 41 ++++---- pkg/bps/pb/bps.proto | 2 +- pkg/bps/registry.go | 56 ---------- pkg/bps/registry_test.go | 1 - 5 files changed, 186 insertions(+), 129 deletions(-) delete mode 100644 pkg/bps/registry.go delete mode 100644 pkg/bps/registry_test.go diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index ad9301732e9..354c75e360c 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -7,7 +7,9 @@ package bps import ( + "bytes" "context" + "crypto/rand" "errors" "fmt" "sync" @@ -16,6 +18,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" ) @@ -32,13 +35,14 @@ var errNotBroker = errors.New("not a broker") // Service is the bps protocol service. type Service struct { - mtx sync.Mutex - fullNode bool - streamer p2p.Streamer - + fullNode bool + mtx sync.Mutex + selfOverlay swarm.Address // this node's overlay. used to verify challenges + streamer p2p.Streamer + cohorts map[string]*cohort // cohort registry - keep track of members, publisher, channels, challenge, streams? // this is only relevant for the actual broker. subscribers don't care of manage this state at all. - registry *CohortRegistry + // registry *CohortRegistry logger log.Logger } @@ -47,8 +51,8 @@ type Service struct { func New(streamer p2p.Streamer, logger log.Logger) *Service { return &Service{ streamer: streamer, - registry: NewRegistry(), - logger: logger.WithName(loggerName).Register(), + // registry: NewRegistry(), + logger: logger.WithName(loggerName).Register(), } } @@ -68,25 +72,25 @@ func (s *Service) Protocol() p2p.ProtocolSpec { // Join onto a topic at the broker at address. This is called on the subscriber. // Returns the challenge and a channel that would send the payloads over it. -func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) ([]byte, chan []byte, error) { +func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) (challenge []byte, rx, tx chan []byte, claim func([]byte), err error) { stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) if err != nil { - return nil, nil, fmt.Errorf("new stream: %w", err) + return nil, nil, nil, nil, fmt.Errorf("new stream: %w", err) } w, r := protobuf.NewWriterAndReader(stream) joinMsg := pb.Join{Topic: topic} if err := w.WriteMsgWithContext(ctx, &joinMsg); err != nil { - return nil, nil, fmt.Errorf("write join msg: %w", err) + return nil, nil, nil, nil, fmt.Errorf("write join msg: %w", err) } joinAck := pb.JoinAck{} if err := r.ReadMsgWithContext(ctx, &joinAck); err != nil { - return nil, nil, fmt.Errorf("read join ack: %w", err) + return nil, nil, nil, nil, fmt.Errorf("read join ack: %w", err) } rxCh := make(chan []byte) - // from now on we expect only to read updates off this stream + // reading should always yield broadcast msgs go func() { defer stream.FullClose() @@ -106,48 +110,46 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) } }() - return joinAck.Challenge, rxCh, nil -} - -// claim a given topic on a broker (at address) with the provided signature. -// this is called on the subsciber that wants to become a publisher. -// returns the write channel used in order to send later payloads. -func (s *Service) Claim(ctx context.Context, address swarm.Address, topic, sig []byte) (chan []byte, error) { - stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) - if err != nil { - return nil, fmt.Errorf("new stream: %w", err) - } - w, _ := protobuf.NewWriterAndReader(stream) - claim := pb.Claim{Sig: topic} - if err := w.WriteMsgWithContext(ctx, &claim); err != nil { - return nil, fmt.Errorf("write claim: %w", err) + // the write loop will only be used in the case of a publisher. + // for readers this is essentially a noop. + tx = make(chan []byte) + claim = func(soc []byte) { + select { + case tx <- soc: + case <-ctx.Done(): + return + } } - // if the claim is wrong we will just get kicked off with a stream reset and the read will fail - ch := make(chan []byte) - - // from now on we expect only to write updates to this stream go func() { defer stream.FullClose() - for { - // we might want to do some input validation to see that the broker isn't tricking us + select { + case cl := <-tx: + claim := pb.Claim{Soc: cl} + if err := w.WriteMsgWithContext(ctx, &claim); err != nil { + s.logger.Error(err, "read join ack") + return + } + case <-ctx.Done(): + return + } + for { select { - case v, ok := <-ch: - // closing the channel causes us to exit and close the stream - if !ok { - return - } - msg := pb.Broadcast{Soc: v} + case bcast := <-tx: + msg := pb.Broadcast{Soc: bcast} if err := w.WriteMsgWithContext(ctx, &msg); err != nil { - s.logger.Error(err, "read join ack") + s.logger.Error(err, "write broadcast") return } + case <-ctx.Done(): + return } } }() - return ch, nil + + return joinAck.Challenge, rxCh, tx, claim, nil } // handler is the protocol handler on the broker. @@ -164,20 +166,24 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er return fmt.Errorf("read sys message: %w", err) } + // add to the cohort and get a channel to receive the broadcasts on + ch, challenge := s.joinCohort(p.Address, join.Topic) // register in the cohort and return the secret // peer is trying to join the cohort. accept and return the challenge - ack := pb.JoinAck{Challenge: []byte{0, 1, 2, 3}} + ack := pb.JoinAck{Challenge: challenge} if err := w.WriteMsgWithContext(ctx, &ack); err != nil { go stream.FullClose() + s.left(p.Address, join.Topic) return fmt.Errorf("write claim: %w", err) } - // add to the cohort and get a channel to receive the broadcasts on - ch := make(chan []byte) // replace with the registry channel later - + defer s.left(p.Address, join.Topic) + var wg sync.WaitGroup // the writer side - reads messages off the publisher channel // and pushes them to the subscriber stream + wg.Add(1) go func() { + defer wg.Done() defer stream.FullClose() for { // await messages, then write to the stream once they come in @@ -195,7 +201,6 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er }() // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off - //_ = claim.Challenge // check if claim signature pk matches the feed owner address on the registry // if it does - this is an atomic swap - the current stream becomes the publisher stream // and the next challenge changes randomly, so that if needed, the publisher can reclaim @@ -205,8 +210,9 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er // once a claim is accepted, we prompte the stream to be a publisher stream // then continuously try to read from it broadcast messages and push them over the // publisher channel - ch1 := make(chan []byte) // replace with the cohort send channel later + wg.Add(1) go func() { + defer wg.Done() defer stream.FullClose() claim := pb.Claim{} if err := r.ReadMsgWithContext(ctx, &claim); err != nil { @@ -215,7 +221,12 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } // do the checks on the claim, if it is wrong then we reset the stream - + ch1, err := s.claim(p.Address, join.Topic, claim.Soc) + if err != nil { + stream.Reset() + return + } + defer close(ch1) for { // await messages, then write to the cohort channel m := pb.Broadcast{} @@ -230,5 +241,113 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } } }() + wg.Wait() return nil } + +var errInvalidProof = errors.New("invalid proof") + +// join/open operation in one - the peer either joins an existing +// cohort or creates one by joining a previously unregistered topic. +// returns the cohort challenge and the channel that data will be sent over +func (s *Service) joinCohort(overlay swarm.Address, topic []byte) (chan []byte, []byte) { + s.mtx.Lock() + defer s.mtx.Unlock() + t := string(topic) + co, ok := s.cohorts[t] + + m := newMember() + if ok { + co.members[overlay.String()] = m + return m.ch, m.challenge + } + members := make(map[string]*member) + members[overlay.String()] = m + s.cohorts[t] = &cohort{ + topic: topic, + members: members, + } + + return m.ch, m.challenge +} + +func (s *Service) left(overlay swarm.Address, topic []byte) { + s.mtx.Lock() + defer s.mtx.Unlock() + t := string(topic) + co, ok := s.cohorts[t] + if ok { + delete(co.members, overlay.String()) + if len(co.members) == 0 { + delete(s.cohorts, t) + } + } +} + +func (s *Service) claim(overlay swarm.Address, topic, socBlob []byte) (chan []byte, error) { + s.mtx.Lock() + defer s.mtx.Unlock() + t := string(topic) + co, ok := s.cohorts[t] + if !ok { + return nil, errors.New("tried to claim non-existent topic") + } + member, ok := co.members[overlay.String()] + if !ok { + return nil, errors.New("no such member") + } + chunk := swarm.NewChunk(swarm.NewAddress(topic), socBlob) + if !soc.Valid(chunk) { + return nil, errInvalidProof + } + innerProof := make([]byte, len(s.selfOverlay.Bytes())+len(member.challenge)) + copy(innerProof, member.challenge) + copy(innerProof[len(member.challenge):], s.selfOverlay.Bytes()) + proofSoc, _ := soc.FromChunk(chunk) + if !bytes.Equal(proofSoc.WrappedChunk().Data()[swarm.SpanSize:], innerProof) { + return nil, errInvalidProof + } + + txCh := make(chan []byte) + go func() { + for v := range txCh { + s.mtx.Lock() + co, ok := s.cohorts[t] + if !ok { + s.mtx.Unlock() + return + } + for _, member := range co.members { + select { + case member.ch <- v: + default: + } + } + s.mtx.Unlock() + } + }() + return txCh, nil +} + +type cohort struct { + topic []byte // topic is the feed topic + owner hash + // lastSeen uint64 // last feed update index, used to prevent replay and circumvent dedup logic (for now) + members map[string]*member +} + +func newMember() *member { + challenge := make([]byte, 32) + if _, err := rand.Read(challenge); err != nil { + panic(err) + } + + return &member{ + ch: make(chan []byte), + challenge: challenge, + } +} + +type member struct { + ch chan []byte + challenge []byte +} diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index 35a5d69fb34..12e27a9601e 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -22,11 +22,6 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package -// Both directions after the handshake: admin -> broker is a publication, broker -> -// subscriber a delivery of the same bytes. The single-owner chunk travels as its -// stored chunk data, opaque to the protocol and validated by the ordinary SOC code -// once the `id` slot has been rewritten as the Frames section says: -// id (32) || signature (65) || span (8, LE) || payload (<= 4096) type Broadcast struct { Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` } @@ -160,7 +155,7 @@ func (m *JoinAck) GetChallenge() []byte { } type Claim struct { - Sig []byte `protobuf:"bytes,1,opt,name=sig,proto3" json:"sig,omitempty"` + Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` } func (m *Claim) Reset() { *m = Claim{} } @@ -196,9 +191,9 @@ func (m *Claim) XXX_DiscardUnknown() { var xxx_messageInfo_Claim proto.InternalMessageInfo -func (m *Claim) GetSig() []byte { +func (m *Claim) GetSoc() []byte { if m != nil { - return m.Sig + return m.Soc } return nil } @@ -213,18 +208,18 @@ func init() { func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 168 bytes of a gzipped FileDescriptorProto + // 162 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0x2a, 0x28, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x4e, 0x2a, 0x28, 0x56, 0x92, 0xe5, 0xe2, 0x74, 0x2a, 0xca, 0x4f, 0x4c, 0x49, 0x4e, 0x2c, 0x2e, 0x11, 0x12, 0xe0, 0x62, 0x2e, 0xce, 0x4f, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0x31, 0x95, 0x64, 0xb8, 0x58, 0xbc, 0xf2, 0x33, 0xf3, 0x84, 0x44, 0xb8, 0x58, 0x4b, 0xf2, 0x0b, 0x32, 0x61, 0x72, 0x10, 0x8e, 0x92, 0x3a, 0x17, 0x3b, 0x48, 0xd6, 0x31, 0x39, 0x5b, 0x48, 0x86, 0x8b, 0x33, 0x39, 0x23, 0x31, 0x27, 0x27, 0x35, 0x2f, 0x3d, - 0x15, 0xaa, 0x08, 0x21, 0xa0, 0x24, 0xc9, 0xc5, 0xea, 0x9c, 0x93, 0x98, 0x99, 0x0b, 0xb6, 0x21, - 0x33, 0x1d, 0x6e, 0x43, 0x66, 0xba, 0x93, 0xcc, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, - 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, - 0x31, 0x44, 0x31, 0x15, 0x24, 0x25, 0xb1, 0x81, 0x9d, 0x6a, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, - 0x75, 0xae, 0x61, 0xc4, 0xb7, 0x00, 0x00, 0x00, + 0x15, 0xaa, 0x08, 0x21, 0xa0, 0x24, 0xc9, 0xc5, 0xea, 0x9c, 0x93, 0x98, 0x99, 0x8b, 0x69, 0x83, + 0x93, 0xcc, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, + 0xb1, 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x44, 0x31, 0x15, 0x24, 0x25, + 0xb1, 0x81, 0x9d, 0x6a, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x2e, 0xa9, 0x0f, 0xdc, 0xb7, 0x00, + 0x00, 0x00, } func (m *Broadcast) Marshal() (dAtA []byte, err error) { @@ -337,10 +332,10 @@ func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Sig) > 0 { - i -= len(m.Sig) - copy(dAtA[i:], m.Sig) - i = encodeVarintBps(dAtA, i, uint64(len(m.Sig))) + if len(m.Soc) > 0 { + i -= len(m.Soc) + copy(dAtA[i:], m.Soc) + i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) i-- dAtA[i] = 0xa } @@ -403,7 +398,7 @@ func (m *Claim) Size() (n int) { } var l int _ = l - l = len(m.Sig) + l = len(m.Soc) if l > 0 { n += 1 + l + sovBps(uint64(l)) } @@ -708,7 +703,7 @@ func (m *Claim) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Sig", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -735,9 +730,9 @@ func (m *Claim) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Sig = append(m.Sig[:0], dAtA[iNdEx:postIndex]...) - if m.Sig == nil { - m.Sig = []byte{} + m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) + if m.Soc == nil { + m.Soc = []byte{} } iNdEx = postIndex default: diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index 0efa47b1365..77ff296043c 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -21,5 +21,5 @@ message JoinAck { } message Claim { - bytes sig = 1; // the challenge signature that takes over the cohort. signs the challenge in JoinAck. + bytes soc = 1; // the soc chunk body to claim the publishing rights } diff --git a/pkg/bps/registry.go b/pkg/bps/registry.go deleted file mode 100644 index 20be39707be..00000000000 --- a/pkg/bps/registry.go +++ /dev/null @@ -1,56 +0,0 @@ -package bps - -import ( - "crypto/rand" - "sync" - - "github.com/ethersphere/bee/v2/pkg/swarm" -) - -func NewRegistry() *CohortRegistry { - return &CohortRegistry{ - cohorts: make(map[string]*cohort), - } -} - -type CohortRegistry struct { - mtx sync.Mutex - cohorts map[string]*cohort -} - -// join/open operation in one - the peer either joins an existing -// cohort or creates one by joining a previously unregistered topic. -// returns the cohort challenge -func (c *CohortRegistry) Join(overlay swarm.Address, topic []byte, ch chan []byte) []byte { - c.mtx.Lock() - defer c.mtx.Unlock() - t := string(topic) - co, ok := c.cohorts[t] - if ok { - co.members[overlay.String()] = ch - return co.challenge - } - members := make(map[string]chan []byte) - challenge := make([]byte, 32) - if _, err := rand.Read(challenge); err != nil { - panic(err) - } - - members[overlay.String()] = ch - c.cohorts[t] = &cohort{ - topic: topic, - members: members, - challenge: challenge, - } - - return challenge -} - -type cohort struct { - // mtx sync.Mutex - topic []byte // topic is the feed topic + owner hash - lastSeen uint64 // last feed update index, used to prevent replay and circumvent dedup logic (for now) - members map[string]chan []byte - publisher swarm.Address - challenge []byte -} diff --git a/pkg/bps/registry_test.go b/pkg/bps/registry_test.go deleted file mode 100644 index 7c30ef52b76..00000000000 --- a/pkg/bps/registry_test.go +++ /dev/null @@ -1 +0,0 @@ -package bps_test From 6f9bed27c5740f3b46dce4a32b2938a4529991f1 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:23:30 -0600 Subject: [PATCH 10/29] chore: tests, improvements (cherry picked from commit 14b89a5d569919555a00441503af4e666aa64178) --- pkg/bps/bps.go | 36 ++++++++++----- pkg/bps/bps_test.go | 107 +++++++++++++++++++++++++------------------- 2 files changed, 85 insertions(+), 58 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 354c75e360c..894dac7372e 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -31,6 +31,11 @@ const ( streamName = "bps" ) +// broadcastBufferSize is the per-member buffer for broadcast messages. +// Delivery is lossy by design (a slow member must not stall the cohort), +// so the buffer absorbs short bursts while the member's stream is busy. +const broadcastBufferSize = 1 + var errNotBroker = errors.New("not a broker") // Service is the bps protocol service. @@ -48,11 +53,13 @@ type Service struct { } // New returns a new bps Service. -func New(streamer p2p.Streamer, logger log.Logger) *Service { +func New(streamer p2p.Streamer, overlay swarm.Address, fullNode bool, logger log.Logger) *Service { return &Service{ - streamer: streamer, - // registry: NewRegistry(), - logger: logger.WithName(loggerName).Register(), + fullNode: fullNode, + selfOverlay: overlay, + streamer: streamer, + cohorts: make(map[string]*cohort), + logger: logger.WithName(loggerName).Register(), } } @@ -88,7 +95,7 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) return nil, nil, nil, nil, fmt.Errorf("read join ack: %w", err) } - rxCh := make(chan []byte) + rxCh := make(chan []byte, broadcastBufferSize) // reading should always yield broadcast msgs go func() { @@ -158,11 +165,13 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er stream.Reset() return errNotBroker } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + defer stream.FullClose() w, r := protobuf.NewWriterAndReader(stream) var join pb.Join if err := r.ReadMsgWithContext(ctx, &join); err != nil { - go stream.FullClose() return fmt.Errorf("read sys message: %w", err) } @@ -172,7 +181,6 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er // peer is trying to join the cohort. accept and return the challenge ack := pb.JoinAck{Challenge: challenge} if err := w.WriteMsgWithContext(ctx, &ack); err != nil { - go stream.FullClose() s.left(p.Address, join.Topic) return fmt.Errorf("write claim: %w", err) } @@ -184,7 +192,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er wg.Add(1) go func() { defer wg.Done() - defer stream.FullClose() + defer cancel() for { // await messages, then write to the stream once they come in select { @@ -213,7 +221,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er wg.Add(1) go func() { defer wg.Done() - defer stream.FullClose() + defer cancel() claim := pb.Claim{} if err := r.ReadMsgWithContext(ctx, &claim); err != nil { s.logger.Error(err, "read claim") @@ -308,6 +316,7 @@ func (s *Service) claim(overlay swarm.Address, topic, socBlob []byte) (chan []by return nil, errInvalidProof } + publisher := member txCh := make(chan []byte) go func() { for v := range txCh { @@ -317,9 +326,12 @@ func (s *Service) claim(overlay swarm.Address, topic, socBlob []byte) (chan []by s.mtx.Unlock() return } - for _, member := range co.members { + for _, m := range co.members { + if m == publisher { + continue + } select { - case member.ch <- v: + case m.ch <- v: default: } } @@ -342,7 +354,7 @@ func newMember() *member { } return &member{ - ch: make(chan []byte), + ch: make(chan []byte, broadcastBufferSize), challenge: challenge, } } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index be126e0d131..fb9b67c6fe6 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -7,83 +7,98 @@ package bps_test import ( "bytes" "context" - "fmt" "testing" + "time" "github.com/ethersphere/bee/v2/pkg/bps" - "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" - "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" "github.com/ethersphere/bee/v2/pkg/p2p/streamtest" + "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" ) -func TestJoin(t *testing.T) { +func TestClaimAndBroadcast(t *testing.T) { t.Parallel() logger := log.Noop - server := bps.New(nil, logger) + brokerAddr := swarm.RandAddress(t) + broker := bps.New(nil, brokerAddr, true, logger) - recorder := streamtest.New( - streamtest.WithProtocols(server.Protocol()), - ) - - client := bps.New(recorder, logger) - - addr := swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c") - topic := []byte{0, 1, 2, 3, 4} - // greeting := "world" + // each client gets its own recorder so that the broker sees distinct peer overlays + newClient := func() *bps.Service { + recorder := streamtest.New( + streamtest.WithProtocols(broker.Protocol()), + streamtest.WithBaseAddr(swarm.RandAddress(t)), + ) + return bps.New(recorder, swarm.RandAddress(t), false, logger) + } + publisher := newClient() + subscriber := newClient() - challenge, _, err := client.Join(context.Background(), addr, topic) + // the public topic is the soc address of the claim: keccak(id | owner) + key, err := crypto.GenerateSecp256k1Key() if err != nil { t.Fatal(err) } - fmt.Println(challenge) - records, err := recorder.Records(addr, "bps", "1.0.0", "bps") + signer := crypto.NewDefaultSigner(key) + owner, err := crypto.NewEthereumAddress(key.PublicKey) if err != nil { t.Fatal(err) } - if l := len(records); l != 1 { - t.Fatalf("got %v records, want %v", l, 2) + id := make([]byte, swarm.HashSize) + copy(id, "bps-test-topic") + topic, err := soc.CreateAddress(id, owner) + if err != nil { + t.Fatal(err) } - record := records[0] - // client -> server: SystemMessage{Join} - messages, err := protobuf.ReadMessages( - bytes.NewReader(record.In()), - func() protobuf.Message { return new(pb.SystemMessage) }, - ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, subRx, _, _, err := subscriber.Join(ctx, brokerAddr, topic.Bytes()) if err != nil { t.Fatal(err) } - if l := len(messages); l != 1 { - t.Fatalf("got %v messages, want %v", l, 1) - } - join := messages[0].(*pb.SystemMessage).GetJoin() - if join == nil { - t.Fatal("expected join message") - } - if !bytes.Equal(join.Topic, topic) { - t.Fatalf("got topic %x, want %x", join.Topic, topic) + + challenge, pubRx, pubTx, claim, err := publisher.Join(ctx, brokerAddr, topic.Bytes()) + if err != nil { + t.Fatal(err) } - // server -> client returns challenge - messages, err = protobuf.ReadMessages( - bytes.NewReader(record.Out()), - func() protobuf.Message { return new(pb.JoinAck) }, - ) + // claim payload: challenge | broker overlay + payload := append(append([]byte{}, challenge...), brokerAddr.Bytes()...) + ch, err := cac.New(payload) + if err != nil { + t.Fatal(err) + } + proof, err := soc.New(id, ch).Sign(signer) if err != nil { t.Fatal(err) } - if l := len(messages); l != 1 { - t.Fatalf("got %v messages, want %v", l, 1) + claim(proof.Data()) + + msg := []byte("hello cohort") + select { + case pubTx <- msg: + case <-time.After(time.Second): + t.Fatal("timed out sending broadcast") } - cl := messages[0].(*pb.JoinAck).Challenge - if cl == nil { - t.Fatal("expected joinack message") + + select { + case got := <-subRx: + if !bytes.Equal(got, msg) { + t.Fatalf("got message %q, want %q", got, msg) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for subscriber to receive broadcast") } - if !bytes.Equal(challenge, cl) { - t.Fatalf("got challenge %x, want %x", cl, challenge) + + select { + case got := <-pubRx: + t.Fatalf("publisher received its own broadcast %q", got) + case <-time.After(100 * time.Millisecond): } } From 2343b6cac0650d5d1317aa2f1860cb0601c396ee Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:36:44 -0600 Subject: [PATCH 11/29] chore: bps api (cherry picked from commit 2d7adcf296f2ad968b96aaa6ed78d07f41ac0d35) --- openapi/Swarm.yaml | 91 ++++++ pkg/api/api.go | 3 + pkg/api/api_test.go | 2 + pkg/api/bps.go | 459 ++++++++++++++++++++++++++++++ pkg/api/bps_test.go | 615 +++++++++++++++++++++++++++++++++++++++++ pkg/api/export_test.go | 16 ++ pkg/api/router.go | 8 + pkg/bps/TODO | 52 ++++ 8 files changed, 1246 insertions(+) create mode 100644 pkg/api/bps.go create mode 100644 pkg/api/bps_test.go create mode 100644 pkg/bps/TODO diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index cce68073b91..9a980336f20 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1049,6 +1049,97 @@ paths: default: description: Default response + "/bps/subscribe/{owner}/{topic}": + get: + summary: Subscribe to a BPS cohort + description: | + WebSocket. The node joins the cohort `keccak256(keccak256("bps-claim" | topic) | owner)` + at `broker` and sends every broadcast as a binary frame with the full SOC bytes + (`id | signature | span | payload`). Broadcasts not signed by `owner` are dropped. + The endpoint is read-only: any client frame closes the socket with 4002. + Close codes: 4002 invalid message, 4003 broker stream closed. + tags: + - BPS + parameters: + - in: path + name: owner + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: true + description: Ethereum address of the cohort owner + - in: path + name: topic + schema: + $ref: "SwarmCommon.yaml#/components/schemas/HexString" + required: true + description: 32-byte cohort topic + - in: query + name: broker + schema: + $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" + required: true + description: Overlay address of the broker node. Must not be this node. + responses: + "200": + description: Establishes a WebSocket subscription for incoming broadcasts + "400": + $ref: "SwarmCommon.yaml#/components/responses/400" + "500": + $ref: "SwarmCommon.yaml#/components/responses/500" + "503": + description: BPS service is not configured + default: + description: Default response + + "/bps/publish/{owner}/{topic}": + get: + summary: Claim and publish to a BPS cohort + description: | + WebSocket. Text frames carry JSON control messages; binary frames carry SOCs. + + 1. Server sends `{"type":"challenge","challenge":"","broker":"","id":""}`. + 2. Client signs a SOC with `id` wrapping the payload `challenge | broker` and sends + `{"type":"claim","signature":""}` within 30 seconds. + 3. Server verifies the claim locally, forwards it to the broker and sends + `{"type":"claim_sent"}`. This does not mean the broker accepted it; a rejection + closes the socket with 4003. + 4. Client sends each broadcast as a binary frame containing a full SOC at the cohort address. + + Close codes: 4001 invalid claim, 4002 invalid message (bad SOC, wrong frame type, + claim timeout), 4003 broker stream closed. + tags: + - BPS + parameters: + - in: path + name: owner + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: true + description: Ethereum address of the cohort owner + - in: path + name: topic + schema: + $ref: "SwarmCommon.yaml#/components/schemas/HexString" + required: true + description: 32-byte cohort topic + - in: query + name: broker + schema: + $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" + required: true + description: Overlay address of the broker node. Must not be this node. + responses: + "200": + description: Establishes a WebSocket subscription for publishing broadcasts + "400": + $ref: "SwarmCommon.yaml#/components/responses/400" + "500": + $ref: "SwarmCommon.yaml#/components/responses/500" + "503": + description: BPS service is not configured + default: + description: Default response + "/soc/{owner}/{id}": post: summary: Upload a Single Owner Chunk diff --git a/pkg/api/api.go b/pkg/api/api.go index 38ac92c048b..79bba870afd 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -166,6 +166,7 @@ type Service struct { resolver resolver.Interface pss pss.Interface gsoc gsoc.Listener + bps BPSService steward steward.Interface logger log.Logger loggerV1 log.Logger @@ -285,6 +286,7 @@ type ExtraOptions struct { Resolver resolver.Interface Pss pss.Interface Gsoc gsoc.Listener + Bps BPSService FeedFactory feeds.Factory Post postage.Service AccessControl accesscontrol.Controller @@ -368,6 +370,7 @@ func (s *Service) Configure(signer crypto.Signer, tracer *tracing.Tracer, o Opti s.resolver = e.Resolver s.pss = e.Pss s.gsoc = e.Gsoc + s.bps = e.Bps s.feedFactory = e.FeedFactory s.post = e.Post s.accesscontrol = e.AccessControl diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 02de8ce6d68..d4d3fe71e9d 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -96,6 +96,7 @@ type testServerOptions struct { Resolver resolver.Interface Pss pss.Interface Gsoc gsoc.Listener + Bps api.BPSService WsPath string WsPingPeriod time.Duration Logger log.Logger @@ -210,6 +211,7 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket. Resolver: o.Resolver, Pss: o.Pss, Gsoc: o.Gsoc, + Bps: o.Bps, FeedFactory: o.Feeds, Post: o.Post, AccessControl: o.AccessControl, diff --git a/pkg/api/bps.go b/pkg/api/bps.go new file mode 100644 index 00000000000..de8f369cef6 --- /dev/null +++ b/pkg/api/bps.go @@ -0,0 +1,459 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package api + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/jsonhttp" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" +) + +const ( + bpsCloseInvalidClaim = 4001 // claim failed local verification or was malformed + bpsCloseInvalidMessage = 4002 // frame violates the endpoint protocol + bpsCloseBrokerGone = 4003 // the p2p stream to the broker ended + + bpsMaxCloseReason = 123 // websocket control frame payload limit minus the code + bpsMaxFrameSize = swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + swarm.ChunkSize +) + +// bpsTopicPrefix domain-separates the cohort SOC id from the owner's other SOCs. +var bpsTopicPrefix = []byte("bps-claim") + +// bpsClaimTimeout bounds how long a publisher may take to send its claim. +var bpsClaimTimeout = 30 * time.Second + +var errBPSBrokerGone = errors.New("broker stream closed") + +// BPSService joins cohorts at a broker. +type BPSService interface { + Join(ctx context.Context, broker, addr swarm.Address) (BPSSession, error) +} + +// BPSSession is a single joined p2p stream to a broker. +type BPSSession interface { + // Challenge is the nonce the broker issued for this stream. + Challenge() []byte + // Messages yields raw SOC bytes broadcast by the broker. + // Implementations must not block when Messages is not drained (the + // publish endpoint never reads it). Closing the channel is treated + // as the stream ending. + Messages() <-chan []byte + // Claim writes the claim SOC to the broker. The broker does not reply. + Claim(ctx context.Context, soc []byte) error + // Publish writes a broadcast SOC to the broker. + Publish(ctx context.Context, soc []byte) error + // Done is closed when the p2p stream ends. + Done() <-chan struct{} + // Err reports why the stream ended. Valid after Done is closed. + Err() error + Close() error +} + +type bpsRequest struct { + owner common.Address + id []byte + addr swarm.Address + broker swarm.Address +} + +type bpsChallengeMessage struct { + Type string `json:"type"` + Challenge string `json:"challenge"` + Broker string `json:"broker"` + ID string `json:"id"` +} + +type bpsClaimMessage struct { + Type string `json:"type"` + Signature string `json:"signature"` +} + +type bpsClaimSentMessage struct { + Type string `json:"type"` +} + +// bpsTopicAddress derives the cohort SOC id and address for owner and topic. +func bpsTopicAddress(owner common.Address, topic []byte) ([]byte, swarm.Address, error) { + id, err := crypto.LegacyKeccak256(append(append([]byte{}, bpsTopicPrefix...), topic...)) + if err != nil { + return nil, swarm.ZeroAddress, err + } + addr, err := soc.CreateAddress(id, owner.Bytes()) + if err != nil { + return nil, swarm.ZeroAddress, err + } + return id, addr, nil +} + +func (s *Service) bpsSubscribeWsHandler(w http.ResponseWriter, r *http.Request) { + logger := s.logger.WithName("bps_subscribe").Build() + req, ok := s.bpsParseRequest(w, r, logger) + if !ok { + return + } + conn, ok := s.bpsUpgrade(w, r, logger) + if !ok { + return + } + s.wsWg.Add(1) + go s.bpsSubscribeWs(conn, req, logger) +} + +func (s *Service) bpsPublishWsHandler(w http.ResponseWriter, r *http.Request) { + logger := s.logger.WithName("bps_publish").Build() + req, ok := s.bpsParseRequest(w, r, logger) + if !ok { + return + } + conn, ok := s.bpsUpgrade(w, r, logger) + if !ok { + return + } + s.wsWg.Add(1) + go s.bpsPublishWs(conn, req, logger) +} + +// bpsParseRequest validates the request before the websocket upgrade. +// It writes the HTTP error response itself and reports whether to continue. +func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger log.Logger) (bpsRequest, bool) { + if s.bps == nil { + jsonhttp.ServiceUnavailable(w, "bps unavailable") + return bpsRequest{}, false + } + + paths := struct { + Owner common.Address `map:"owner" validate:"required"` + Topic []byte `map:"topic" validate:"required,len=32"` + }{} + if response := s.mapStructure(mux.Vars(r), &paths); response != nil { + response("invalid path params", logger, w) + return bpsRequest{}, false + } + + queries := struct { + Broker swarm.Address `map:"broker"` + }{} + if response := s.mapStructure(r.URL.Query(), &queries); response != nil { + response("invalid query params", logger, w) + return bpsRequest{}, false + } + if queries.Broker.IsZero() { + jsonhttp.BadRequest(w, "missing broker") + return bpsRequest{}, false + } + if len(queries.Broker.Bytes()) != swarm.HashSize { + jsonhttp.BadRequest(w, "invalid broker") + return bpsRequest{}, false + } + if s.overlay != nil && queries.Broker.Equal(*s.overlay) { + jsonhttp.BadRequest(w, "broker cannot be this node") + return bpsRequest{}, false + } + + id, addr, err := bpsTopicAddress(paths.Owner, paths.Topic) + if err != nil { + logger.Debug("derive topic address failed", "error", err) + jsonhttp.InternalServerError(w, "derive topic address failed") + return bpsRequest{}, false + } + return bpsRequest{owner: paths.Owner, id: id, addr: addr, broker: queries.Broker}, true +} + +func (s *Service) bpsUpgrade(w http.ResponseWriter, r *http.Request, logger log.Logger) (*websocket.Conn, bool) { + upgrader := websocket.Upgrader{ + ReadBufferSize: swarm.ChunkSize, + WriteBufferSize: swarm.ChunkSize, + CheckOrigin: s.checkOrigin, + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + logger.Debug("upgrade failed", "error", err) + logger.Error(nil, "upgrade failed") + jsonhttp.InternalServerError(w, "upgrade failed") + return nil, false + } + conn.SetReadLimit(bpsMaxFrameSize) + return conn, true +} + +func (s *Service) bpsWrite(conn *websocket.Conn, messageType int, data []byte) error { + if err := conn.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil { + return err + } + return conn.WriteMessage(messageType, data) +} + +func (s *Service) bpsWriteJSON(conn *websocket.Conn, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + return s.bpsWrite(conn, websocket.TextMessage, b) +} + +// bpsClose sends a close frame with code and reason. Errors are ignored +// since the connection is torn down right after. +func (s *Service) bpsClose(conn *websocket.Conn, code int, reason string) { + if len(reason) > bpsMaxCloseReason { + reason = strings.ToValidUTF8(reason[:bpsMaxCloseReason], "") + } + _ = s.bpsWrite(conn, websocket.CloseMessage, websocket.FormatCloseMessage(code, reason)) +} + +func bpsErrReason(err error) string { + if err == nil { + return errBPSBrokerGone.Error() + } + return err.Error() +} + +type bpsFrame struct { + typ int + data []byte +} + +// bpsReadFrames reads client frames until the connection errors or quit is +// closed. The returned gone channel is closed when reading stops. +func bpsReadFrames(conn *websocket.Conn, quit <-chan struct{}) (<-chan bpsFrame, <-chan struct{}) { + frames := make(chan bpsFrame) + gone := make(chan struct{}) + go func() { + defer close(gone) + for { + typ, data, err := conn.ReadMessage() + if err != nil { + return + } + select { + case frames <- bpsFrame{typ: typ, data: data}: + case <-quit: + return + } + } + }() + return frames, gone +} + +func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger log.Logger) { + defer s.wsWg.Done() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + select { + case <-s.quit: + cancel() + case <-ctx.Done(): + } + }() + quit := make(chan struct{}) + ticker := time.NewTicker(s.WsPingPeriod) + defer func() { + ticker.Stop() + close(quit) + cancel() + _ = conn.Close() + }() + + sess, err := s.bps.Join(ctx, req.broker, req.addr) + if err != nil { + logger.Debug("join failed", "broker", req.broker, "error", err) + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) + return + } + defer sess.Close() + + frames, gone := bpsReadFrames(conn, quit) + + for { + select { + case data, ok := <-sess.Messages(): + if !ok { + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) + return + } + if !soc.Valid(swarm.NewChunk(req.addr, data)) { + logger.Debug("dropping invalid broadcast", "address", req.addr) + continue + } + if err := s.bpsWrite(conn, websocket.BinaryMessage, data); err != nil { + logger.Debug("write broadcast failed", "error", err) + return + } + case <-frames: + s.bpsClose(conn, bpsCloseInvalidMessage, "subscribe endpoint is read-only") + return + case <-sess.Done(): + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) + return + case <-gone: + return + case <-s.quit: + _ = s.bpsWrite(conn, websocket.CloseMessage, []byte{}) + return + case <-ticker.C: + if err := s.bpsWrite(conn, websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// bpsVerifyClaim checks a claim frame the same way the broker will and +// returns the claim SOC bytes, or a close code and reason on failure. +func bpsVerifyClaim(f bpsFrame, req bpsRequest, challenge []byte) ([]byte, int, string) { + if f.typ != websocket.TextMessage { + return nil, bpsCloseInvalidMessage, "expected claim" + } + var m bpsClaimMessage + if err := json.Unmarshal(f.data, &m); err != nil { + return nil, bpsCloseInvalidClaim, "malformed claim" + } + if m.Type != "claim" { + return nil, bpsCloseInvalidMessage, "expected claim" + } + sig, err := hex.DecodeString(m.Signature) + if err != nil || len(sig) != swarm.SocSignatureSize { + return nil, bpsCloseInvalidClaim, "invalid signature" + } + payload := make([]byte, 0, len(challenge)+swarm.HashSize) + payload = append(append(payload, challenge...), req.broker.Bytes()...) + ch, err := cac.New(payload) + if err != nil { + return nil, bpsCloseInvalidClaim, "invalid claim payload" + } + sc, err := soc.NewSigned(req.id, ch, req.owner.Bytes(), sig) + if err != nil { + return nil, bpsCloseInvalidClaim, "invalid claim" + } + chunk, err := sc.Chunk() + if err != nil { + return nil, bpsCloseInvalidClaim, "invalid claim" + } + // NewSigned does not verify the signature; Valid recovers the signer + // and checks that it maps to the cohort address. + if !chunk.Address().Equal(req.addr) || !soc.Valid(chunk) { + return nil, bpsCloseInvalidClaim, "claim verification failed" + } + return chunk.Data(), 0, "" +} + +func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log.Logger) { + defer s.wsWg.Done() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + select { + case <-s.quit: + cancel() + case <-ctx.Done(): + } + }() + quit := make(chan struct{}) + ticker := time.NewTicker(s.WsPingPeriod) + defer func() { + ticker.Stop() + close(quit) + cancel() + _ = conn.Close() + }() + + sess, err := s.bps.Join(ctx, req.broker, req.addr) + if err != nil { + logger.Debug("join failed", "broker", req.broker, "error", err) + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) + return + } + defer sess.Close() + + challenge := sess.Challenge() + if len(challenge) != swarm.HashSize { + logger.Debug("invalid challenge length", "broker", req.broker, "length", len(challenge)) + s.bpsClose(conn, bpsCloseBrokerGone, "invalid challenge") + return + } + if err := s.bpsWriteJSON(conn, bpsChallengeMessage{ + Type: "challenge", + Challenge: hex.EncodeToString(challenge), + Broker: req.broker.String(), + ID: hex.EncodeToString(req.id), + }); err != nil { + logger.Debug("write challenge failed", "error", err) + return + } + + claimTimer := time.NewTimer(bpsClaimTimeout) + defer claimTimer.Stop() + + frames, gone := bpsReadFrames(conn, quit) + claimed := false + + for { + select { + case f := <-frames: + if !claimed { + claim, code, reason := bpsVerifyClaim(f, req, challenge) + if code != 0 { + s.bpsClose(conn, code, reason) + return + } + if err := sess.Claim(ctx, claim); err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) + return + } + claimed = true + claimTimer.Stop() + if err := s.bpsWriteJSON(conn, bpsClaimSentMessage{Type: "claim_sent"}); err != nil { + logger.Debug("write claim_sent failed", "error", err) + return + } + continue + } + if f.typ != websocket.BinaryMessage { + s.bpsClose(conn, bpsCloseInvalidMessage, "expected binary soc") + return + } + if !soc.Valid(swarm.NewChunk(req.addr, f.data)) { + s.bpsClose(conn, bpsCloseInvalidMessage, "invalid soc") + return + } + if err := sess.Publish(ctx, f.data); err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) + return + } + case <-claimTimer.C: + if !claimed { + s.bpsClose(conn, bpsCloseInvalidMessage, "claim timeout") + return + } + case <-sess.Done(): + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) + return + case <-gone: + return + case <-s.quit: + _ = s.bpsWrite(conn, websocket.CloseMessage, []byte{}) + return + case <-ticker.C: + if err := s.bpsWrite(conn, websocket.PingMessage, nil); err != nil { + return + } + } + } +} diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go new file mode 100644 index 00000000000..f21d6191c71 --- /dev/null +++ b/pkg/api/bps_test.go @@ -0,0 +1,615 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package api_test + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/url" + "sync" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/jsonhttp" + "github.com/ethersphere/bee/v2/pkg/jsonhttp/jsonhttptest" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/bee/v2/pkg/util/testutil" + "github.com/gorilla/websocket" +) + +type fakeBPS struct { + sess *fakeSession + joinErr error + + mu sync.Mutex + broker swarm.Address + addr swarm.Address +} + +func (f *fakeBPS) Join(_ context.Context, broker, addr swarm.Address) (api.BPSSession, error) { + f.mu.Lock() + f.broker, f.addr = broker, addr + f.mu.Unlock() + if f.joinErr != nil { + return nil, f.joinErr + } + return f.sess, nil +} + +type fakeSession struct { + challenge []byte + msgs chan []byte + done chan struct{} + claims chan []byte + published chan []byte + closed chan struct{} + + claimErr error + publishErr error + + doneOnce sync.Once + closeOnce sync.Once + err error +} + +func newFakeSession() *fakeSession { + challenge := make([]byte, 32) + copy(challenge, "bps-test-challenge") + return &fakeSession{ + challenge: challenge, + msgs: make(chan []byte), + done: make(chan struct{}), + claims: make(chan []byte, 1), + published: make(chan []byte, 8), + closed: make(chan struct{}), + } +} + +func (f *fakeSession) Challenge() []byte { return f.challenge } +func (f *fakeSession) Messages() <-chan []byte { return f.msgs } +func (f *fakeSession) Done() <-chan struct{} { return f.done } +func (f *fakeSession) Err() error { return f.err } + +func (f *fakeSession) Claim(_ context.Context, b []byte) error { + if f.claimErr != nil { + return f.claimErr + } + f.claims <- b + return nil +} + +func (f *fakeSession) Publish(_ context.Context, b []byte) error { + if f.publishErr != nil { + return f.publishErr + } + f.published <- b + return nil +} + +func (f *fakeSession) Close() error { + f.closeOnce.Do(func() { close(f.closed) }) + return nil +} + +// end simulates the broker stream ending with err. +func (f *fakeSession) end(err error) { + f.doneOnce.Do(func() { + f.err = err + close(f.done) + }) +} + +type bpsFixture struct { + client *http.Client + listener string + bps *fakeBPS + sess *fakeSession + signer crypto.Signer + owner []byte + topic []byte + id []byte + addr swarm.Address + broker swarm.Address + self swarm.Address +} + +func newBPSFixture(t *testing.T) *bpsFixture { + t.Helper() + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + owner, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + topic := make([]byte, 32) + copy(topic, "bps-test-topic") + id, err := crypto.LegacyKeccak256(append([]byte("bps-claim"), topic...)) + if err != nil { + t.Fatal(err) + } + addr, err := soc.CreateAddress(id, owner.Bytes()) + if err != nil { + t.Fatal(err) + } + sess := newFakeSession() + f := &bpsFixture{ + bps: &fakeBPS{sess: sess}, + sess: sess, + signer: signer, + owner: owner.Bytes(), + topic: topic, + id: id, + addr: addr, + broker: swarm.RandAddress(t), + self: swarm.RandAddress(t), + } + f.client, _, f.listener, _, _ = newTestServer(t, testServerOptions{ + Bps: f.bps, + Overlay: f.self, + WsPingPeriod: 60 * time.Second, + }) + return f +} + +func (f *bpsFixture) path(endpoint string) string { + return fmt.Sprintf("/bps/%s/%s/%s", endpoint, hex.EncodeToString(f.owner), hex.EncodeToString(f.topic)) +} + +func (f *bpsFixture) dial(t *testing.T, endpoint string) *websocket.Conn { + t.Helper() + u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path(endpoint), RawQuery: "broker=" + f.broker.String()} + conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + t.Fatalf("dial %s: %v", u.String(), err) + } + testutil.CleanupCloser(t, conn) + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + return conn +} + +// signedSOC returns full SOC bytes at f.addr wrapping payload. +func (f *bpsFixture) signedSOC(t *testing.T, signer crypto.Signer, payload []byte) []byte { + t.Helper() + ch, err := cac.New(payload) + if err != nil { + t.Fatal(err) + } + c, err := soc.New(f.id, ch).Sign(signer) + if err != nil { + t.Fatal(err) + } + return c.Data() +} + +// claimSignature signs the claim payload challenge|broker with signer. +func (f *bpsFixture) claimSignature(t *testing.T, signer crypto.Signer, challenge []byte) string { + t.Helper() + payload := append(append([]byte{}, challenge...), f.broker.Bytes()...) + data := f.signedSOC(t, signer, payload) + return hex.EncodeToString(data[swarm.HashSize : swarm.HashSize+swarm.SocSignatureSize]) +} + +func expectClose(t *testing.T, conn *websocket.Conn, code int) { + t.Helper() + for { + _, _, err := conn.ReadMessage() + if err == nil { + continue + } + var ce *websocket.CloseError + if !errors.As(err, &ce) { + t.Fatalf("want close error with code %d, got %v", code, err) + } + if ce.Code != code { + t.Fatalf("want close code %d, got %d (%q)", code, ce.Code, ce.Text) + } + return + } +} + +func TestBPSPreUpgrade(t *testing.T) { + t.Parallel() + + f := newBPSFixture(t) + nilClient, _, _, _, _ := newTestServer(t, testServerOptions{}) + owner := hex.EncodeToString(f.owner) + topic := hex.EncodeToString(f.topic) + + for _, endpoint := range []string{"subscribe", "publish"} { + t.Run(endpoint, func(t *testing.T) { + t.Parallel() + base := fmt.Sprintf("/bps/%s/%s/%s", endpoint, owner, topic) + + jsonhttptest.Request(t, f.client, http.MethodGet, base, http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "missing broker", + }), + ) + jsonhttptest.Request(t, f.client, http.MethodGet, base+"?broker=zz", http.StatusBadRequest) + jsonhttptest.Request(t, f.client, http.MethodGet, + fmt.Sprintf("/bps/%s/%s/%s?broker=%s", endpoint, "zz", topic, f.broker), http.StatusBadRequest) + jsonhttptest.Request(t, f.client, http.MethodGet, + fmt.Sprintf("/bps/%s/%s/%s?broker=%s", endpoint, owner, "abcd", f.broker), http.StatusBadRequest) + jsonhttptest.Request(t, f.client, http.MethodGet, base+"?broker=abcd", http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "invalid broker", + }), + ) + jsonhttptest.Request(t, f.client, http.MethodGet, base+"?broker="+f.self.String(), http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "broker cannot be this node", + }), + ) + jsonhttptest.Request(t, nilClient, http.MethodGet, base+"?broker="+f.broker.String(), http.StatusServiceUnavailable) + }) + } +} + +func TestBPSSubscribe(t *testing.T) { + t.Parallel() + + t.Run("forwards valid soc", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "subscribe") + msg := f.signedSOC(t, f.signer, []byte("hello cohort")) + f.sess.msgs <- msg + typ, got, err := conn.ReadMessage() + if err != nil { + t.Fatal(err) + } + if typ != websocket.BinaryMessage || !bytes.Equal(got, msg) { + t.Fatalf("got type %d %x, want binary %x", typ, got, msg) + } + f.bps.mu.Lock() + gotAddr, gotBroker := f.bps.addr, f.bps.broker + f.bps.mu.Unlock() + if !gotAddr.Equal(f.addr) || !gotBroker.Equal(f.broker) { + t.Fatalf("join got addr %s broker %s, want %s %s", gotAddr, gotBroker, f.addr, f.broker) + } + }) + + t.Run("drops invalid soc", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "subscribe") + other, _ := crypto.GenerateSecp256k1Key() + f.sess.msgs <- f.signedSOC(t, crypto.NewDefaultSigner(other), []byte("forged")) + f.sess.msgs <- []byte("garbage") + valid := f.signedSOC(t, f.signer, []byte("real")) + f.sess.msgs <- valid + _, got, err := conn.ReadMessage() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, valid) { + t.Fatalf("got %x, want only the valid soc %x", got, valid) + } + }) + + t.Run("client frame closes 4002", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "subscribe") + if err := conn.WriteMessage(websocket.BinaryMessage, []byte("nope")); err != nil { + t.Fatal(err) + } + expectClose(t, conn, api.BPSCloseInvalidMessage) + <-f.sess.closed + }) + + t.Run("broker gone closes 4003", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "subscribe") + f.sess.end(errors.New("stream reset")) + expectClose(t, conn, api.BPSCloseBrokerGone) + <-f.sess.closed + }) + + t.Run("join error closes 4003", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.bps.joinErr = errors.New("broker unreachable") + conn := f.dial(t, "subscribe") + expectClose(t, conn, api.BPSCloseBrokerGone) + }) +} + +func TestBPSSubscribeMessagesClosed(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "subscribe") + close(f.sess.msgs) + expectClose(t, conn, api.BPSCloseBrokerGone) + <-f.sess.closed +} + +// waitSessionClosed fails the test if the session is not closed in time. +func waitSessionClosed(t *testing.T, f *bpsFixture) { + t.Helper() + select { + case <-f.sess.closed: + case <-time.After(5 * time.Second): + t.Fatal("session not closed") + } +} + +// readChallenge reads the challenge frame and checks its fields. +func readChallenge(t *testing.T, f *bpsFixture, conn *websocket.Conn) []byte { + t.Helper() + var m struct { + Type string `json:"type"` + Challenge string `json:"challenge"` + Broker string `json:"broker"` + ID string `json:"id"` + } + if err := conn.ReadJSON(&m); err != nil { + t.Fatal(err) + } + if m.Type != "challenge" || m.Broker != f.broker.String() || m.ID != hex.EncodeToString(f.id) { + t.Fatalf("unexpected challenge message %+v", m) + } + challenge, err := hex.DecodeString(m.Challenge) + if err != nil || !bytes.Equal(challenge, f.sess.challenge) { + t.Fatalf("challenge %q, want %x", m.Challenge, f.sess.challenge) + } + return challenge +} + +func sendClaim(t *testing.T, conn *websocket.Conn, sig string) { + t.Helper() + if err := conn.WriteJSON(map[string]string{"type": "claim", "signature": sig}); err != nil { + t.Fatal(err) + } +} + +func expectClaimSent(t *testing.T, conn *websocket.Conn) { + t.Helper() + var m struct { + Type string `json:"type"` + } + if err := conn.ReadJSON(&m); err != nil { + t.Fatal(err) + } + if m.Type != "claim_sent" { + t.Fatalf("got %q, want claim_sent", m.Type) + } +} + +// claimed dials publish and completes the claim handshake. +func claimed(t *testing.T, f *bpsFixture) *websocket.Conn { + t.Helper() + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) + sendClaim(t, conn, f.claimSignature(t, f.signer, challenge)) + expectClaimSent(t, conn) + return conn +} + +func TestBPSPublish(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := claimed(t, f) + + claim := <-f.sess.claims + if !soc.Valid(swarm.NewChunk(f.addr, claim)) { + t.Fatal("claim passed to session is not a valid soc at the topic address") + } + sc, err := soc.FromChunk(swarm.NewChunk(f.addr, claim)) + if err != nil { + t.Fatal(err) + } + wantPayload := append(append([]byte{}, f.sess.challenge...), f.broker.Bytes()...) + if got := sc.WrappedChunk().Data()[swarm.SpanSize:]; !bytes.Equal(got, wantPayload) { + t.Fatalf("claim payload %x, want %x", got, wantPayload) + } + + msg := f.signedSOC(t, f.signer, []byte("broadcast")) + if err := conn.WriteMessage(websocket.BinaryMessage, msg); err != nil { + t.Fatal(err) + } + select { + case got := <-f.sess.published: + if !bytes.Equal(got, msg) { + t.Fatalf("published %x, want %x", got, msg) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for publish") + } +} + +func TestBPSPublishClaimErrors(t *testing.T) { + t.Parallel() + + other, _ := crypto.GenerateSecp256k1Key() + otherSigner := crypto.NewDefaultSigner(other) + + for _, tc := range []struct { + name string + send func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) + code int + }{ + { + name: "wrong signer", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + sendClaim(t, conn, f.claimSignature(t, otherSigner, challenge)) + }, + code: api.BPSCloseInvalidClaim, + }, + { + name: "wrong challenge", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { + sendClaim(t, conn, f.claimSignature(t, f.signer, make([]byte, 32))) + }, + code: api.BPSCloseInvalidClaim, + }, + { + name: "malformed json", + send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { + _ = conn.WriteMessage(websocket.TextMessage, []byte("{not json")) + }, + code: api.BPSCloseInvalidClaim, + }, + { + name: "short signature", + send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { + sendClaim(t, conn, "abcd") + }, + code: api.BPSCloseInvalidClaim, + }, + { + name: "0x-prefixed signature", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + sendClaim(t, conn, "0x"+f.claimSignature(t, f.signer, challenge)) + }, + code: api.BPSCloseInvalidClaim, + }, + { + name: "wrong message type", + send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { + _ = conn.WriteJSON(map[string]string{"type": "hello"}) + }, + code: api.BPSCloseInvalidMessage, + }, + { + name: "binary before claim", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { + _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, f.signer, []byte("early"))) + }, + code: api.BPSCloseInvalidMessage, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) + tc.send(t, f, conn, challenge) + expectClose(t, conn, tc.code) + waitSessionClosed(t, f) + if len(f.sess.claims) != 0 { + t.Fatal("rejected claim reached the session") + } + }) + } +} + +func TestBPSPublishAfterClaimErrors(t *testing.T) { + t.Parallel() + + t.Run("invalid soc closes 4002", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := claimed(t, f) + other, _ := crypto.GenerateSecp256k1Key() + _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, crypto.NewDefaultSigner(other), []byte("forged"))) + expectClose(t, conn, api.BPSCloseInvalidMessage) + if len(f.sess.published) != 0 { + t.Fatal("invalid soc was published") + } + }) + + t.Run("text frame closes 4002", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := claimed(t, f) + _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"type":"claim"}`)) + expectClose(t, conn, api.BPSCloseInvalidMessage) + }) + + t.Run("broker gone closes 4003", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := claimed(t, f) + f.sess.end(errors.New("stream reset")) + expectClose(t, conn, api.BPSCloseBrokerGone) + waitSessionClosed(t, f) + }) + + t.Run("claim write error closes 4003", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.sess.claimErr = errors.New("write failed") + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) + sendClaim(t, conn, f.claimSignature(t, f.signer, challenge)) + expectClose(t, conn, api.BPSCloseBrokerGone) + }) + + t.Run("publish error closes 4003", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.sess.publishErr = errors.New("write failed") + conn := claimed(t, f) + _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, f.signer, []byte("x"))) + expectClose(t, conn, api.BPSCloseBrokerGone) + }) +} + +func TestBPSPublishClientGoneBeforeClaim(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "publish") + readChallenge(t, f, conn) + // Send a close frame rather than Close, which the dial cleanup would + // report as an error on the already closed connection. + _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + select { + case <-f.sess.closed: + case <-time.After(5 * time.Second): + t.Fatal("session not closed after client went away") + } +} + +func TestBPSPublishOversizedFrame(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := claimed(t, f) + _ = conn.WriteMessage(websocket.BinaryMessage, make([]byte, 5000)) + expectClose(t, conn, websocket.CloseMessageTooBig) + waitSessionClosed(t, f) +} + +// Not parallel: mutates the package-level claim timeout. +func TestBPSPublishClaimTimeout(t *testing.T) { + restore := api.SetBPSClaimTimeout(100 * time.Millisecond) + defer restore() + f := newBPSFixture(t) + conn := f.dial(t, "publish") + readChallenge(t, f, conn) + expectClose(t, conn, api.BPSCloseInvalidMessage) +} + +func TestBPSPublishInvalidChallengeLength(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.sess.challenge = make([]byte, 100) + conn := f.dial(t, "publish") + // expectClose skips data frames, so check explicitly that the first + // frame is the close and no challenge was sent. + _, _, err := conn.ReadMessage() + var ce *websocket.CloseError + if !errors.As(err, &ce) { + t.Fatalf("want close error, got %v", err) + } + if ce.Code != api.BPSCloseBrokerGone || ce.Text != "invalid challenge" { + t.Fatalf("got close %d %q", ce.Code, ce.Text) + } + waitSessionClosed(t, f) +} diff --git a/pkg/api/export_test.go b/pkg/api/export_test.go index c96b9e81ae8..173d9c94656 100644 --- a/pkg/api/export_test.go +++ b/pkg/api/export_test.go @@ -5,6 +5,8 @@ package api import ( + "time" + "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/gorilla/mux" @@ -162,3 +164,17 @@ func NewParseError(entry, value string, cause error) error { const RootPath = rootPath func (s *Service) Router() *mux.Router { return s.router } + +const ( + BPSCloseInvalidClaim = bpsCloseInvalidClaim + BPSCloseInvalidMessage = bpsCloseInvalidMessage + BPSCloseBrokerGone = bpsCloseBrokerGone +) + +// SetBPSClaimTimeout overrides the claim timeout and returns a restore func. +// Callers must not run in parallel with other bps tests. +func SetBPSClaimTimeout(d time.Duration) func() { + prev := bpsClaimTimeout + bpsClaimTimeout = d + return func() { bpsClaimTimeout = prev } +} diff --git a/pkg/api/router.go b/pkg/api/router.go index 7c11fd04b58..bac2014c087 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -392,6 +392,14 @@ func (s *Service) mountAPI() { ), }) + handle("/bps/subscribe/{owner}/{topic}", jsonhttp.MethodHandler{ + "GET": http.HandlerFunc(s.bpsSubscribeWsHandler), + }) + + handle("/bps/publish/{owner}/{topic}", jsonhttp.MethodHandler{ + "GET": http.HandlerFunc(s.bpsPublishWsHandler), + }) + handle("/tags", jsonhttp.MethodHandler{ http.MethodGet: http.HandlerFunc(s.listTagsHandler), http.MethodPost: web.ChainHandlers( diff --git a/pkg/bps/TODO b/pkg/bps/TODO new file mode 100644 index 00000000000..c4e0b659d75 --- /dev/null +++ b/pkg/bps/TODO @@ -0,0 +1,52 @@ +● Here's what's still open, from the earlier reviews plus a few smaller things I noticed while working in the code. + + Protocol and security + + 1. Publisher tracking (#7): add cohort.publisher and allow only one publisher per cohort. On takeover, close the old publisher's stream. Doing + the check, the swap and the eviction under one lock also stops two claims from racing each other. + 2. Owner checks (#8): + - Store the owner recovered from the claim in the cohort. + - The broker should check that each broadcast is a valid SOC signed by that owner. + - Subscribers should check too, which means deciding how they learn the owner. + 3. Members keyed by stream, not overlay (#9): a second join from the same peer currently overwrites the first, and either stream's left then + removes the other. Fixing this also closes the case where the publisher stalls sending on ch1 after the fan-out goroutine has exited. + 4. Domain-separated topic ID: use id = keccak("bps-claim" | t) so the claim chunk can't overwrite the owner's own SOC or GSOC in Swarm. This is a + client-side convention; a helper function would enforce it. + 5. Limits on cohort creation: reject Join.Topic values that aren't 32 bytes, and cap cohorts per peer and in total. + 6. Replay and ordering of broadcasts: lastSeen is commented out, so nothing stops a broker from re-sending old messages. + + Client (Join) + + 7. The first message on tx is always sent as the claim. Give the claim its own path or a state flag. + 8. Wrong log messages: several write errors are logged as "read join ack". + + Code quality + + 9. newMember panics if rand.Read fails; it should return an error. + 10. The ad-hoc errors.New messages should be bps:-prefixed sentinel errors. + 11. soc.FromChunk's error is ignored. It can't fail once Valid passes, but it deserves a comment. + 12. The fan-out goroutine takes the service-wide s.mtx for every message. A lock per cohort will be needed once there are many cohorts. + 13. Stale comments: + - The package doc still describes a "hello-world wire protocol used to greet other peers". + - The handler comment still says the "next challenge changes randomly", but nonces are now per member. + + Integration + + 14. Wiring: nothing in pkg/node/node.go calls bps.New yet. It needs swarmAddress and o.FullNodeMode, and the protocol has to be registered with + p2p. + + Tests + + 15. Claim cases not covered yet: + - a claim signed for a different broker + - a stale nonce, e.g. from an earlier stream + - a bad signature, or a signer other than the owner + - publisher takeover, once #1 is done + - a subscriber leaving and being removed + - an unverified broadcast being rejected, once #2 is done + + Housekeeping + + 16. Unrelated diff: regenerated .pb.go files in 12 unrelated packages are still modified in the working tree. They're safe to revert. + + I'd do 3, then 1 and 2 together (tracking the publisher and storing the owner are really one change), then 7, then 14. From 0e0b6f323490bef85ad4cb6ff262c1396854e2e6 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:23:06 -0600 Subject: [PATCH 12/29] feat(bps): cohort spec with binding and principal, identity per stream, node wiring --- openapi/Swarm.yaml | 12 ++ pkg/api/api.go | 5 +- pkg/api/bps.go | 68 ++++---- pkg/bps/binding.go | 81 ++++++++++ pkg/bps/bps.go | 301 +++++++++++++++++++++++++++------- pkg/bps/pb/bps.pb.go | 375 +++++++++++++++++++++++++------------------ pkg/bps/pb/bps.proto | 21 ++- pkg/node/node.go | 9 ++ 8 files changed, 622 insertions(+), 250 deletions(-) create mode 100644 pkg/bps/binding.go diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 9a980336f20..ab77e340332 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1079,6 +1079,12 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" required: true description: Overlay address of the broker node. Must not be this node. + - in: query + name: identity + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: true + description: Ethereum address identifying the joiner. responses: "200": description: Establishes a WebSocket subscription for incoming broadcasts @@ -1128,6 +1134,12 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" required: true description: Overlay address of the broker node. Must not be this node. + - in: query + name: identity + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: true + description: Ethereum address identifying the joiner. responses: "200": description: Establishes a WebSocket subscription for publishing broadcasts diff --git a/pkg/api/api.go b/pkg/api/api.go index 79bba870afd..fce6a7821e5 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/accesscontrol" "github.com/ethersphere/bee/v2/pkg/accounting" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/pipeline" @@ -166,7 +167,7 @@ type Service struct { resolver resolver.Interface pss pss.Interface gsoc gsoc.Listener - bps BPSService + bps bps.Interface steward steward.Interface logger log.Logger loggerV1 log.Logger @@ -286,7 +287,7 @@ type ExtraOptions struct { Resolver resolver.Interface Pss pss.Interface Gsoc gsoc.Listener - Bps BPSService + Bps bps.Interface FeedFactory feeds.Factory Post postage.Service AccessControl accesscontrol.Controller diff --git a/pkg/api/bps.go b/pkg/api/bps.go index de8f369cef6..f802cb2bed3 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -14,6 +14,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/jsonhttp" @@ -41,36 +42,13 @@ var bpsClaimTimeout = 30 * time.Second var errBPSBrokerGone = errors.New("broker stream closed") -// BPSService joins cohorts at a broker. -type BPSService interface { - Join(ctx context.Context, broker, addr swarm.Address) (BPSSession, error) -} - -// BPSSession is a single joined p2p stream to a broker. -type BPSSession interface { - // Challenge is the nonce the broker issued for this stream. - Challenge() []byte - // Messages yields raw SOC bytes broadcast by the broker. - // Implementations must not block when Messages is not drained (the - // publish endpoint never reads it). Closing the channel is treated - // as the stream ending. - Messages() <-chan []byte - // Claim writes the claim SOC to the broker. The broker does not reply. - Claim(ctx context.Context, soc []byte) error - // Publish writes a broadcast SOC to the broker. - Publish(ctx context.Context, soc []byte) error - // Done is closed when the p2p stream ends. - Done() <-chan struct{} - // Err reports why the stream ended. Valid after Done is closed. - Err() error - Close() error -} - type bpsRequest struct { - owner common.Address - id []byte - addr swarm.Address - broker swarm.Address + owner common.Address + id []byte + topic []byte + addr swarm.Address + broker swarm.Address + identity []byte } type bpsChallengeMessage struct { @@ -148,7 +126,8 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger } queries := struct { - Broker swarm.Address `map:"broker"` + Broker swarm.Address `map:"broker"` + Identity []byte `map:"identity" validate:"required,len=20"` }{} if response := s.mapStructure(r.URL.Query(), &queries); response != nil { response("invalid query params", logger, w) @@ -173,7 +152,14 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger jsonhttp.InternalServerError(w, "derive topic address failed") return bpsRequest{}, false } - return bpsRequest{owner: paths.Owner, id: id, addr: addr, broker: queries.Broker}, true + return bpsRequest{ + owner: paths.Owner, + id: id, + topic: paths.Topic, + addr: addr, + broker: queries.Broker, + identity: queries.Identity, + }, true } func (s *Service) bpsUpgrade(w http.ResponseWriter, r *http.Request, logger log.Logger) (*websocket.Conn, bool) { @@ -271,7 +257,13 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo _ = conn.Close() }() - sess, err := s.bps.Join(ctx, req.broker, req.addr) + sess, err := s.bps.Join(ctx, bps.JoinRequest{ + Broker: req.broker, + Binding: bps.BindingFeed, + Topic: req.topic, + Principal: req.owner.Bytes(), + Identity: req.identity, + }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) @@ -288,10 +280,6 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) return } - if !soc.Valid(swarm.NewChunk(req.addr, data)) { - logger.Debug("dropping invalid broadcast", "address", req.addr) - continue - } if err := s.bpsWrite(conn, websocket.BinaryMessage, data); err != nil { logger.Debug("write broadcast failed", "error", err) return @@ -374,7 +362,13 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. _ = conn.Close() }() - sess, err := s.bps.Join(ctx, req.broker, req.addr) + sess, err := s.bps.Join(ctx, bps.JoinRequest{ + Broker: req.broker, + Binding: bps.BindingFeed, + Topic: req.topic, + Principal: req.owner.Bytes(), + Identity: req.identity, + }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) diff --git a/pkg/bps/binding.go b/pkg/bps/binding.go new file mode 100644 index 00000000000..f2482bda946 --- /dev/null +++ b/pkg/bps/binding.go @@ -0,0 +1,81 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "errors" + "fmt" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// claimIDPrefix domain-separates the cohort's claim SOC id from the +// principal's other single owner chunks. +var claimIDPrefix = []byte("bps-claim") + +// errUnsupportedBinding is returned for a Join naming a topic binding this +// node does not implement. +var errUnsupportedBinding = errors.New("bps: unsupported topic binding") + +// errClaimNotQualified is returned when a claim SOC does not satisfy its +// cohort's topic binding. +var errClaimNotQualified = errors.New("bps: claim does not qualify") + +// binding decides how a cohort's publisher claim is verified. The topic +// binding named in Join picks the implementation; feed is the only one for now. +type binding interface { + // claimAddress returns the SOC address a valid claim must have for the + // cohort's topic and governing principal. + claimAddress(topic, principal []byte) (swarm.Address, error) + // verifyClaim checks the decoded claim's owner and signed payload. It + // returns nil when the claim promotes the stream to publisher. + verifyClaim(s *soc.SOC, principal, challenge, broker, topic []byte) error +} + +// bindingFor returns the binding rules for b, or errUnsupportedBinding. +func bindingFor(b pb.TopicBinding) (binding, error) { + switch b { + case pb.TopicBinding_BINDING_FEED: + return feedBinding{}, nil + } + return nil, fmt.Errorf("%s: %w", b, errUnsupportedBinding) +} + +// feedBinding verifies the publisher claim for a feed topic. The claim is a +// SOC with id keccak256("bps-claim" | topic), owned by the governing principal, +// whose wrapped payload signs the broker overlay and feed topic together with +// the per-member challenge. +type feedBinding struct{} + +func (feedBinding) claimAddress(topic, principal []byte) (swarm.Address, error) { + id, err := feedClaimID(topic) + if err != nil { + return swarm.ZeroAddress, err + } + return soc.CreateAddress(id, principal) +} + +func (feedBinding) verifyClaim(s *soc.SOC, principal, challenge, broker, topic []byte) error { + if !bytes.Equal(s.OwnerAddress(), principal) { + return fmt.Errorf("claim owner is not the principal: %w", errClaimNotQualified) + } + innerProof := make([]byte, 0, len(challenge)+len(broker)+len(topic)) + innerProof = append(innerProof, challenge...) + innerProof = append(innerProof, broker...) + innerProof = append(innerProof, topic...) + if !bytes.Equal(s.WrappedChunk().Data()[swarm.SpanSize:], innerProof) { + return fmt.Errorf("claim payload mismatch: %w", errClaimNotQualified) + } + return nil +} + +// feedClaimID derives the claim SOC id for a feed topic. +func feedClaimID(topic []byte) ([]byte, error) { + return crypto.LegacyKeccak256(append(append([]byte{}, claimIDPrefix...), topic...)) +} diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 894dac7372e..53756ade28b 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -15,6 +15,7 @@ import ( "sync" "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" @@ -36,7 +37,11 @@ const ( // so the buffer absorbs short bursts while the member's stream is busy. const broadcastBufferSize = 1 -var errNotBroker = errors.New("not a broker") +var ( + errNotBroker = errors.New("not a broker") + errJoinRejected = errors.New("bps: join rejected") + errSessionClosed = errors.New("bps: session closed") +) // Service is the bps protocol service. type Service struct { @@ -77,65 +82,142 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } } -// Join onto a topic at the broker at address. This is called on the subscriber. -// Returns the challenge and a channel that would send the payloads over it. -func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) (challenge []byte, rx, tx chan []byte, claim func([]byte), err error) { - stream, err := s.streamer.NewStream(ctx, address, nil, protocolName, protocolVersion, streamName) +// Interface is the bps surface consumed by the API. It is satisfied by +// *Service and exists so consumers can depend on behaviour, not the concrete +// implementation. +type Interface interface { + // Join joins a topic's cohort at a broker and returns the live session. + Join(ctx context.Context, req JoinRequest) (Session, error) +} + +// Session is a single joined p2p stream to a broker. +type Session interface { + // Challenge is the nonce the broker issued for this stream. + Challenge() []byte + // Messages yields raw SOC bytes broadcast by the broker. The channel is + // closed when the stream ends. + Messages() <-chan []byte + // Claim writes the publisher claim SOC to the broker. The broker does not + // reply. + Claim(ctx context.Context, soc []byte) error + // Publish writes a broadcast SOC to the broker. + Publish(ctx context.Context, soc []byte) error + // Done is closed when the p2p stream ends. + Done() <-chan struct{} + // Err reports why the stream ended. Valid after Done is closed. + Err() error + // Close closes the stream. + Close() error +} + +// TopicBinding is the topic binding carried by Join, wrapped so consumers do +// not depend on the generated protobuf package. +type TopicBinding int + +const ( + BindingUndefined TopicBinding = iota + BindingFeed +) + +// proto maps the wrapped binding to its wire value. +func (b TopicBinding) proto() pb.TopicBinding { + switch b { + case BindingFeed: + return pb.TopicBinding_BINDING_FEED + default: + return pb.TopicBinding_BINDING_UNDEFINED + } +} + +// JoinRequest describes a cohort join. Identity is the joiner's address, +// injected from the API rather than taken from the peer address, so a node can +// join on behalf of a client. +type JoinRequest struct { + Broker swarm.Address // where to join the topic + Binding TopicBinding // which kind of topic binding + Topic []byte // the topic + Principal []byte // the "owner/admin" + Identity []byte // the joining node's identity +} + +// Join onto a topic at the broker. This is called on the subscriber. +func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { + stream, err := s.streamer.NewStream(ctx, req.Broker, nil, protocolName, protocolVersion, streamName) if err != nil { - return nil, nil, nil, nil, fmt.Errorf("new stream: %w", err) + return nil, fmt.Errorf("new stream: %w", err) } + ctx, cancel := context.WithCancel(ctx) + w, r := protobuf.NewWriterAndReader(stream) - joinMsg := pb.Join{Topic: topic} + joinMsg := pb.Join{ + Topic: req.Topic, + Binding: req.Binding.proto(), + Principal: req.Principal, + Identity: req.Identity, + } if err := w.WriteMsgWithContext(ctx, &joinMsg); err != nil { - return nil, nil, nil, nil, fmt.Errorf("write join msg: %w", err) + cancel() + stream.Reset() + return nil, fmt.Errorf("write join msg: %w", err) } joinAck := pb.JoinAck{} if err := r.ReadMsgWithContext(ctx, &joinAck); err != nil { - return nil, nil, nil, nil, fmt.Errorf("read join ack: %w", err) + cancel() + stream.Reset() + return nil, fmt.Errorf("read join ack: %w", err) + } + if joinAck.Status != pb.Status_STATUS_OK { + cancel() + stream.Reset() + return nil, fmt.Errorf("join %s: %w", joinAck.Status, errJoinRejected) } - rxCh := make(chan []byte, broadcastBufferSize) + sess := &session{ + ctx: ctx, + cancel: cancel, + challenge: joinAck.Challenge, + rx: make(chan []byte, broadcastBufferSize), + tx: make(chan []byte), + done: make(chan struct{}), + } // reading should always yield broadcast msgs go func() { defer stream.FullClose() + var readErr error + defer func() { sess.finish(readErr) }() + for { msg := pb.Broadcast{} if err := r.ReadMsgWithContext(ctx, &msg); err != nil { - s.logger.Error(err, "read join ack") + if ctx.Err() == nil { + s.logger.Error(err, "read broadcast") + } + readErr = err return } // we might want to do some input validation to see that the broker isn't tricking us select { - case rxCh <- msg.Soc: + case sess.rx <- msg.Soc: default: } - } }() // the write loop will only be used in the case of a publisher. // for readers this is essentially a noop. - tx = make(chan []byte) - claim = func(soc []byte) { - select { - case tx <- soc: - case <-ctx.Done(): - return - } - } go func() { defer stream.FullClose() select { - case cl := <-tx: - claim := pb.Claim{Soc: cl} + case cl := <-sess.tx: + claim := pb.Broadcast{Soc: cl} if err := w.WriteMsgWithContext(ctx, &claim); err != nil { - s.logger.Error(err, "read join ack") + s.logger.Error(err, "write claim") return } case <-ctx.Done(): @@ -144,7 +226,7 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) for { select { - case bcast := <-tx: + case bcast := <-sess.tx: msg := pb.Broadcast{Soc: bcast} if err := w.WriteMsgWithContext(ctx, &msg); err != nil { s.logger.Error(err, "write broadcast") @@ -156,7 +238,59 @@ func (s *Service) Join(ctx context.Context, address swarm.Address, topic []byte) } }() - return joinAck.Challenge, rxCh, tx, claim, nil + return sess, nil +} + +// session is the concrete Session implementation. +type session struct { + ctx context.Context + cancel context.CancelFunc + challenge []byte + rx chan []byte + tx chan []byte + done chan struct{} + err error + once sync.Once +} + +func (s *session) Challenge() []byte { return s.challenge } +func (s *session) Messages() <-chan []byte { return s.rx } +func (s *session) Done() <-chan struct{} { return s.done } +func (s *session) Err() error { return s.err } +func (s *session) Close() error { + s.cancel() + return nil +} + +func (s *session) Claim(ctx context.Context, soc []byte) error { + select { + case s.tx <- soc: + return nil + case <-s.done: + return errSessionClosed + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *session) Publish(ctx context.Context, soc []byte) error { + select { + case s.tx <- soc: + return nil + case <-s.done: + return errSessionClosed + case <-ctx.Done(): + return ctx.Err() + } +} + +// finish records the terminal error and signals stream completion exactly once. +func (s *session) finish(err error) { + s.once.Do(func() { + s.err = err + close(s.rx) + close(s.done) + }) } // handler is the protocol handler on the broker. @@ -175,17 +309,32 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er return fmt.Errorf("read sys message: %w", err) } + if err := validateJoin(&join); err != nil { + if werr := w.WriteMsgWithContext(ctx, &pb.JoinAck{Status: pb.Status_STATUS_REJECTED}); werr != nil { + return fmt.Errorf("write join ack: %w", werr) + } + stream.Reset() + return err + } + // add to the cohort and get a channel to receive the broadcasts on - ch, challenge := s.joinCohort(p.Address, join.Topic) + ch, challenge, err := s.joinCohort(&join) + if err != nil { + if werr := w.WriteMsgWithContext(ctx, &pb.JoinAck{Status: pb.Status_STATUS_REJECTED}); werr != nil { + return fmt.Errorf("write join ack: %w", werr) + } + stream.Reset() + return err + } // register in the cohort and return the secret // peer is trying to join the cohort. accept and return the challenge - ack := pb.JoinAck{Challenge: challenge} + ack := pb.JoinAck{Challenge: challenge, Status: pb.Status_STATUS_OK} if err := w.WriteMsgWithContext(ctx, &ack); err != nil { - s.left(p.Address, join.Topic) + s.left(join.Identity, join.Topic) return fmt.Errorf("write claim: %w", err) } - defer s.left(p.Address, join.Topic) + defer s.left(join.Identity, join.Topic) var wg sync.WaitGroup // the writer side - reads messages off the publisher channel // and pushes them to the subscriber stream @@ -222,14 +371,14 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er go func() { defer wg.Done() defer cancel() - claim := pb.Claim{} + claim := pb.Broadcast{} if err := r.ReadMsgWithContext(ctx, &claim); err != nil { s.logger.Error(err, "read claim") return } // do the checks on the claim, if it is wrong then we reset the stream - ch1, err := s.claim(p.Address, join.Topic, claim.Soc) + ch1, err := s.claim(&join, claim.Soc) if err != nil { stream.Reset() return @@ -253,67 +402,103 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er return nil } -var errInvalidProof = errors.New("invalid proof") +var ( + errInvalidProof = errors.New("bps: invalid proof") + errInvalidTopic = errors.New("bps: invalid topic") + errInvalidPrincipal = errors.New("bps: invalid principal") + errInvalidIdentity = errors.New("bps: invalid identity") + errCohortMismatch = errors.New("bps: cohort binding mismatch") +) + +// validateJoin rejects a Join naming malformed or unsupported fields. +func validateJoin(join *pb.Join) error { + if len(join.Topic) != swarm.HashSize { + return errInvalidTopic + } + if len(join.Principal) != crypto.AddressSize { + return errInvalidPrincipal + } + if len(join.Identity) != crypto.AddressSize { + return errInvalidIdentity + } + if _, err := bindingFor(join.Binding); err != nil { + return err + } + return nil +} // join/open operation in one - the peer either joins an existing // cohort or creates one by joining a previously unregistered topic. // returns the cohort challenge and the channel that data will be sent over -func (s *Service) joinCohort(overlay swarm.Address, topic []byte) (chan []byte, []byte) { +func (s *Service) joinCohort(join *pb.Join) (chan []byte, []byte, error) { s.mtx.Lock() defer s.mtx.Unlock() - t := string(topic) + t := string(join.Topic) co, ok := s.cohorts[t] - m := newMember() + m := newMember(join.Identity) if ok { - co.members[overlay.String()] = m - return m.ch, m.challenge + if co.binding != join.Binding || !bytes.Equal(co.principal, join.Principal) { + return nil, nil, errCohortMismatch + } + co.members[string(join.Identity)] = m + return m.ch, m.challenge, nil } members := make(map[string]*member) - members[overlay.String()] = m + members[string(join.Identity)] = m s.cohorts[t] = &cohort{ - topic: topic, - members: members, + topic: join.Topic, + binding: join.Binding, + principal: join.Principal, + members: members, } - return m.ch, m.challenge + return m.ch, m.challenge, nil } -func (s *Service) left(overlay swarm.Address, topic []byte) { +func (s *Service) left(identity, topic []byte) { s.mtx.Lock() defer s.mtx.Unlock() t := string(topic) co, ok := s.cohorts[t] if ok { - delete(co.members, overlay.String()) + delete(co.members, string(identity)) if len(co.members) == 0 { delete(s.cohorts, t) } } } -func (s *Service) claim(overlay swarm.Address, topic, socBlob []byte) (chan []byte, error) { +func (s *Service) claim(join *pb.Join, socBlob []byte) (chan []byte, error) { s.mtx.Lock() defer s.mtx.Unlock() - t := string(topic) + t := string(join.Topic) co, ok := s.cohorts[t] if !ok { return nil, errors.New("tried to claim non-existent topic") } - member, ok := co.members[overlay.String()] + member, ok := co.members[string(join.Identity)] if !ok { return nil, errors.New("no such member") } - chunk := swarm.NewChunk(swarm.NewAddress(topic), socBlob) + b, err := bindingFor(co.binding) + if err != nil { + return nil, err + } + addr, err := b.claimAddress(co.topic, co.principal) + if err != nil { + return nil, fmt.Errorf("claim address: %w", err) + } + chunk := swarm.NewChunk(addr, socBlob) if !soc.Valid(chunk) { return nil, errInvalidProof } - innerProof := make([]byte, len(s.selfOverlay.Bytes())+len(member.challenge)) - copy(innerProof, member.challenge) - copy(innerProof[len(member.challenge):], s.selfOverlay.Bytes()) - proofSoc, _ := soc.FromChunk(chunk) - if !bytes.Equal(proofSoc.WrappedChunk().Data()[swarm.SpanSize:], innerProof) { - return nil, errInvalidProof + proofSoc, err := soc.FromChunk(chunk) + if err != nil { + return nil, fmt.Errorf("claim soc: %w", err) + } + if err := b.verifyClaim(proofSoc, co.principal, member.challenge, s.selfOverlay.Bytes(), co.topic); err != nil { + return nil, err } publisher := member @@ -342,24 +527,28 @@ func (s *Service) claim(overlay swarm.Address, topic, socBlob []byte) (chan []by } type cohort struct { - topic []byte // topic is the feed topic + owner hash + topic []byte // topic is the feed topic + binding pb.TopicBinding + principal []byte // governing identity allowed to publish // lastSeen uint64 // last feed update index, used to prevent replay and circumvent dedup logic (for now) members map[string]*member } -func newMember() *member { +func newMember(identity []byte) *member { challenge := make([]byte, 32) if _, err := rand.Read(challenge); err != nil { panic(err) } return &member{ + identity: identity, ch: make(chan []byte, broadcastBufferSize), challenge: challenge, } } type member struct { + identity []byte ch chan []byte challenge []byte } diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index 12e27a9601e..c0e2696e9a3 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -22,7 +22,64 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package +type TopicBinding int32 + +const ( + TopicBinding_BINDING_UNDEFINED TopicBinding = 0 + TopicBinding_BINDING_FEED TopicBinding = 1 +) + +var TopicBinding_name = map[int32]string{ + 0: "BINDING_UNDEFINED", + 1: "BINDING_FEED", +} + +var TopicBinding_value = map[string]int32{ + "BINDING_UNDEFINED": 0, + "BINDING_FEED": 1, +} + +func (x TopicBinding) String() string { + return proto.EnumName(TopicBinding_name, int32(x)) +} + +func (TopicBinding) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{0} +} + +type Status int32 + +const ( + Status_STATUS_UNDEFINED Status = 0 + Status_STATUS_OK Status = 1 + Status_STATUS_FULL Status = 2 + Status_STATUS_REJECTED Status = 3 +) + +var Status_name = map[int32]string{ + 0: "STATUS_UNDEFINED", + 1: "STATUS_OK", + 2: "STATUS_FULL", + 3: "STATUS_REJECTED", +} + +var Status_value = map[string]int32{ + "STATUS_UNDEFINED": 0, + "STATUS_OK": 1, + "STATUS_FULL": 2, + "STATUS_REJECTED": 3, +} + +func (x Status) String() string { + return proto.EnumName(Status_name, int32(x)) +} + +func (Status) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{1} +} + type Broadcast struct { + // either service or broadcast Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` } @@ -67,7 +124,11 @@ func (m *Broadcast) GetSoc() []byte { } type Join struct { - Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` + Principal []byte `protobuf:"bytes,3,opt,name=principal,proto3" json:"principal,omitempty"` + // ============================ + Identity []byte `protobuf:"bytes,4,opt,name=identity,proto3" json:"identity,omitempty"` } func (m *Join) Reset() { *m = Join{} } @@ -110,8 +171,30 @@ func (m *Join) GetTopic() []byte { return nil } +func (m *Join) GetBinding() TopicBinding { + if m != nil { + return m.Binding + } + return TopicBinding_BINDING_UNDEFINED +} + +func (m *Join) GetPrincipal() []byte { + if m != nil { + return m.Principal + } + return nil +} + +func (m *Join) GetIdentity() []byte { + if m != nil { + return m.Identity + } + return nil +} + type JoinAck struct { Challenge []byte `protobuf:"bytes,1,opt,name=challenge,proto3" json:"challenge,omitempty"` + Status Status `protobuf:"varint,2,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` } func (m *JoinAck) Reset() { *m = JoinAck{} } @@ -154,72 +237,46 @@ func (m *JoinAck) GetChallenge() []byte { return nil } -type Claim struct { - Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` -} - -func (m *Claim) Reset() { *m = Claim{} } -func (m *Claim) String() string { return proto.CompactTextString(m) } -func (*Claim) ProtoMessage() {} -func (*Claim) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{3} -} -func (m *Claim) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Claim) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Claim.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Claim) XXX_Merge(src proto.Message) { - xxx_messageInfo_Claim.Merge(m, src) -} -func (m *Claim) XXX_Size() int { - return m.Size() -} -func (m *Claim) XXX_DiscardUnknown() { - xxx_messageInfo_Claim.DiscardUnknown(m) -} - -var xxx_messageInfo_Claim proto.InternalMessageInfo - -func (m *Claim) GetSoc() []byte { +func (m *JoinAck) GetStatus() Status { if m != nil { - return m.Soc + return m.Status } - return nil + return Status_STATUS_UNDEFINED } func init() { + proto.RegisterEnum("bps.TopicBinding", TopicBinding_name, TopicBinding_value) + proto.RegisterEnum("bps.Status", Status_name, Status_value) proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") proto.RegisterType((*Join)(nil), "bps.Join") proto.RegisterType((*JoinAck)(nil), "bps.JoinAck") - proto.RegisterType((*Claim)(nil), "bps.Claim") } func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 162 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0x2a, 0x28, 0xd6, - 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x4e, 0x2a, 0x28, 0x56, 0x92, 0xe5, 0xe2, 0x74, 0x2a, - 0xca, 0x4f, 0x4c, 0x49, 0x4e, 0x2c, 0x2e, 0x11, 0x12, 0xe0, 0x62, 0x2e, 0xce, 0x4f, 0x96, 0x60, - 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0x31, 0x95, 0x64, 0xb8, 0x58, 0xbc, 0xf2, 0x33, 0xf3, 0x84, - 0x44, 0xb8, 0x58, 0x4b, 0xf2, 0x0b, 0x32, 0x61, 0x72, 0x10, 0x8e, 0x92, 0x3a, 0x17, 0x3b, 0x48, - 0xd6, 0x31, 0x39, 0x5b, 0x48, 0x86, 0x8b, 0x33, 0x39, 0x23, 0x31, 0x27, 0x27, 0x35, 0x2f, 0x3d, - 0x15, 0xaa, 0x08, 0x21, 0xa0, 0x24, 0xc9, 0xc5, 0xea, 0x9c, 0x93, 0x98, 0x99, 0x8b, 0x69, 0x83, - 0x93, 0xcc, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, - 0xb1, 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x44, 0x31, 0x15, 0x24, 0x25, - 0xb1, 0x81, 0x9d, 0x6a, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x2e, 0xa9, 0x0f, 0xdc, 0xb7, 0x00, - 0x00, 0x00, + // 328 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x31, 0x4f, 0xc2, 0x40, + 0x18, 0x86, 0x7b, 0x14, 0xc1, 0x7e, 0xa0, 0x1c, 0x9f, 0x98, 0x34, 0x06, 0x1b, 0x82, 0x0b, 0xc1, + 0x84, 0x41, 0x07, 0x67, 0x6a, 0x0f, 0x03, 0x92, 0x9a, 0xb4, 0x65, 0x71, 0x31, 0x6d, 0x21, 0x78, + 0x91, 0xb4, 0x17, 0x7a, 0x0e, 0xce, 0xfe, 0x01, 0x7f, 0x96, 0x23, 0xa3, 0xa3, 0x81, 0x3f, 0x62, + 0x5a, 0x8a, 0xe8, 0x76, 0xef, 0xf3, 0xde, 0xbd, 0x79, 0x92, 0x03, 0x2d, 0x10, 0x49, 0x4f, 0x2c, + 0x63, 0x19, 0xa3, 0x1a, 0x88, 0xa4, 0x7d, 0x0e, 0x9a, 0xb9, 0x8c, 0xfd, 0x69, 0xe8, 0x27, 0x12, + 0x29, 0xa8, 0x49, 0x1c, 0xea, 0xa4, 0x45, 0x3a, 0x55, 0x27, 0x3d, 0xb6, 0xdf, 0x09, 0x14, 0x47, + 0x31, 0x8f, 0xb0, 0x01, 0x07, 0x32, 0x16, 0x7c, 0x57, 0x6e, 0x03, 0x5e, 0x42, 0x39, 0xe0, 0xd1, + 0x94, 0x47, 0x73, 0xbd, 0xd0, 0x22, 0x9d, 0xe3, 0xab, 0x7a, 0x2f, 0xdd, 0xf7, 0xd2, 0xd2, 0xdc, + 0x16, 0xce, 0xee, 0x06, 0x36, 0x41, 0x13, 0x4b, 0x1e, 0x85, 0x5c, 0xf8, 0x0b, 0x5d, 0xcd, 0x66, + 0xf6, 0x00, 0xcf, 0xe0, 0x90, 0x4f, 0x67, 0x91, 0xe4, 0xf2, 0x4d, 0x2f, 0x66, 0xe5, 0x6f, 0x6e, + 0x8f, 0xa1, 0x9c, 0x4a, 0xf4, 0xc3, 0x97, 0x74, 0x24, 0x7c, 0xf6, 0x17, 0x8b, 0x59, 0x34, 0x9f, + 0xe5, 0x2e, 0x7b, 0x80, 0x17, 0x50, 0x4a, 0xa4, 0x2f, 0x5f, 0x93, 0x5c, 0xa7, 0x92, 0xe9, 0xb8, + 0x19, 0x72, 0xf2, 0xaa, 0x7b, 0x03, 0xd5, 0xbf, 0x82, 0x78, 0x0a, 0x75, 0x73, 0x68, 0x5b, 0x43, + 0xfb, 0xee, 0x69, 0x62, 0x5b, 0x6c, 0x30, 0xb4, 0x99, 0x45, 0x15, 0xa4, 0x50, 0xdd, 0xe1, 0x01, + 0x63, 0x16, 0x25, 0x5d, 0x17, 0x4a, 0xdb, 0x29, 0x6c, 0x00, 0x75, 0xbd, 0xbe, 0x37, 0x71, 0xff, + 0xbd, 0x38, 0x02, 0x2d, 0xa7, 0x0f, 0xf7, 0x94, 0x60, 0x0d, 0x2a, 0x79, 0x1c, 0x4c, 0xc6, 0x63, + 0x5a, 0xc0, 0x13, 0xa8, 0xe5, 0xc0, 0x61, 0x23, 0x76, 0xeb, 0x31, 0x8b, 0xaa, 0x66, 0xf3, 0x73, + 0x6d, 0x90, 0xd5, 0xda, 0x20, 0xdf, 0x6b, 0x83, 0x7c, 0x6c, 0x0c, 0x65, 0xb5, 0x31, 0x94, 0xaf, + 0x8d, 0xa1, 0x3c, 0x16, 0x44, 0x10, 0x94, 0xb2, 0xaf, 0xba, 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, + 0x32, 0x9a, 0x54, 0x7b, 0xb7, 0x01, 0x00, 0x00, } func (m *Broadcast) Marshal() (dAtA []byte, err error) { @@ -272,6 +329,25 @@ func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = encodeVarintBps(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x22 + } + if len(m.Principal) > 0 { + i -= len(m.Principal) + copy(dAtA[i:], m.Principal) + i = encodeVarintBps(dAtA, i, uint64(len(m.Principal))) + i-- + dAtA[i] = 0x1a + } + if m.Binding != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x10 + } if len(m.Topic) > 0 { i -= len(m.Topic) copy(dAtA[i:], m.Topic) @@ -302,6 +378,11 @@ func (m *JoinAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Status != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Status)) + i-- + dAtA[i] = 0x10 + } if len(m.Challenge) > 0 { i -= len(m.Challenge) copy(dAtA[i:], m.Challenge) @@ -312,36 +393,6 @@ func (m *JoinAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *Claim) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Claim) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Soc) > 0 { - i -= len(m.Soc) - copy(dAtA[i:], m.Soc) - i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - func encodeVarintBps(dAtA []byte, offset int, v uint64) int { offset -= sovBps(v) base := offset @@ -376,32 +427,33 @@ func (m *Join) Size() (n int) { if l > 0 { n += 1 + l + sovBps(uint64(l)) } - return n -} - -func (m *JoinAck) Size() (n int) { - if m == nil { - return 0 + if m.Binding != 0 { + n += 1 + sovBps(uint64(m.Binding)) } - var l int - _ = l - l = len(m.Challenge) + l = len(m.Principal) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + l = len(m.Identity) if l > 0 { n += 1 + l + sovBps(uint64(l)) } return n } -func (m *Claim) Size() (n int) { +func (m *JoinAck) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Soc) + l = len(m.Challenge) if l > 0 { n += 1 + l + sovBps(uint64(l)) } + if m.Status != 0 { + n += 1 + sovBps(uint64(m.Status)) + } return n } @@ -561,62 +613,62 @@ func (m *Join) Unmarshal(dAtA []byte) error { m.Topic = []byte{} } iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipBps(dAtA[iNdEx:]) - if err != nil { - return err + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) } - if skippy < 0 { - return ErrInvalidLengthBps + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= TopicBinding(b&0x7F) << shift + if b < 0x80 { + break + } } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthBps + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Principal", wireType) } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *JoinAck) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps + if byteLen < 0 { + return ErrInvalidLengthBps } - if iNdEx >= l { + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break + m.Principal = append(m.Principal[:0], dAtA[iNdEx:postIndex]...) + if m.Principal == nil { + m.Principal = []byte{} } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: JoinAck: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: JoinAck: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: + iNdEx = postIndex + case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -643,9 +695,9 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) - if m.Challenge == nil { - m.Challenge = []byte{} + m.Identity = append(m.Identity[:0], dAtA[iNdEx:postIndex]...) + if m.Identity == nil { + m.Identity = []byte{} } iNdEx = postIndex default: @@ -672,7 +724,7 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { } return nil } -func (m *Claim) Unmarshal(dAtA []byte) error { +func (m *JoinAck) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -695,15 +747,15 @@ func (m *Claim) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Claim: wiretype end group for non-group") + return fmt.Errorf("proto: JoinAck: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Claim: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: JoinAck: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -730,11 +782,30 @@ func (m *Claim) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) - if m.Soc == nil { - m.Soc = []byte{} + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} } iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= Status(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipBps(dAtA[iNdEx:]) diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index 77ff296043c..d112c17d114 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -9,17 +9,32 @@ package bps; option go_package = "pb"; message Broadcast { + // either service or broadcast bytes soc = 1; } +enum TopicBinding { + BINDING_UNDEFINED = 0; + BINDING_FEED = 1; +} + message Join { bytes topic = 1; + TopicBinding binding = 2; // which kind of topic binding + bytes principal = 3; // the "owner" + // ============================ + bytes identity = 4; // who am i +} + +enum Status { + STATUS_UNDEFINED = 0; + STATUS_OK = 1; + STATUS_FULL = 2; + STATUS_REJECTED = 3; } message JoinAck { bytes challenge = 1; // the challenge to sign in order to claim the cohort's publishing role stream + Status status = 2; } -message Claim { - bytes soc = 1; // the soc chunk body to claim the publishing rights -} diff --git a/pkg/node/node.go b/pkg/node/node.go index b9a93612e3e..ca56e7af25f 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -31,6 +31,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/accounting" "github.com/ethersphere/bee/v2/pkg/addressbook" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/config" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/feeds/factory" @@ -1106,6 +1107,13 @@ func NewBee( b.pssCloser = pssService b.gsocCloser = gsocService + bpsService := bps.New(p2ps, swarmAddress, o.FullNodeMode, logger) + if o.FullNodeMode { + if err = p2ps.AddProtocol(bpsService.Protocol()); err != nil { + return nil, fmt.Errorf("bps protocol: %w", err) + } + } + validStamp := postage.ValidStamp(batchStore) // metrics exposed on the status protocol @@ -1381,6 +1389,7 @@ func NewBee( Resolver: multiResolver, Pss: pssService, Gsoc: gsocService, + Bps: bpsService, FeedFactory: feedFactory, Post: post, AccessControl: accesscontrol, From 505dbeaa0b521925b24a903fc677c7f04105c232 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:43:23 -0600 Subject: [PATCH 13/29] feat(bps): align with SWIP-74 rev 8 (BPS-lite) (#5644) (cherry picked from commit 0adf7046c75f6c51790ff3557e5c44276a401d24) --- openapi/Swarm.yaml | 45 ++- pkg/api/api_test.go | 3 +- pkg/api/bps.go | 196 ++++----- pkg/api/bps_test.go | 336 +++++++--------- pkg/api/export_test.go | 12 +- pkg/api/status_test.go | 1 - pkg/bps/TODO | 52 --- pkg/bps/binding.go | 81 ---- pkg/bps/bps.go | 892 +++++++++++++++++++++++++---------------- pkg/bps/bps_test.go | 544 ++++++++++++++++++++++--- pkg/bps/export_test.go | 17 + pkg/bps/id.go | 111 +++++ pkg/bps/metrics.go | 88 ++++ pkg/bps/pb/bps.pb.go | 715 ++++++++++++++++++++++++--------- pkg/bps/pb/bps.proto | 55 ++- pkg/node/node.go | 6 +- 16 files changed, 2087 insertions(+), 1067 deletions(-) delete mode 100644 pkg/bps/TODO delete mode 100644 pkg/bps/binding.go create mode 100644 pkg/bps/export_test.go create mode 100644 pkg/bps/id.go create mode 100644 pkg/bps/metrics.go diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index ab77e340332..1947e6b5ed5 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1053,9 +1053,12 @@ paths: get: summary: Subscribe to a BPS cohort description: | - WebSocket. The node joins the cohort `keccak256(keccak256("bps-claim" | topic) | owner)` - at `broker` and sends every broadcast as a binary frame with the full SOC bytes - (`id | signature | span | payload`). Broadcasts not signed by `owner` are dropped. + WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` + at `broker` as `identity` and sends every update as a binary frame + `challenge (32) | index (8, big-endian) | soc`, where `soc` is the full single-owner + chunk (`id | signature | span | payload`) signed by `owner` with + `id = keccak256(keccak256(topic | challenge) | index)`. Updates that do not verify, + or whose index is below the subscriber's cursor, are dropped. The endpoint is read-only: any client frame closes the socket with 4002. Close codes: 4002 invalid message, 4003 broker stream closed. tags: @@ -1085,6 +1088,12 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" required: true description: Ethereum address identifying the joiner. + - in: query + name: cursor + schema: + type: integer + required: false + description: Lowest update index to deliver. Defaults to 0. responses: "200": description: Establishes a WebSocket subscription for incoming broadcasts @@ -1099,20 +1108,22 @@ paths: "/bps/publish/{owner}/{topic}": get: - summary: Claim and publish to a BPS cohort + summary: Publish to a BPS cohort as its principal description: | - WebSocket. Text frames carry JSON control messages; binary frames carry SOCs. + WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` + at `broker` declaring `owner` as its address. - 1. Server sends `{"type":"challenge","challenge":"","broker":"","id":""}`. - 2. Client signs a SOC with `id` wrapping the payload `challenge | broker` and sends - `{"type":"claim","signature":""}` within 30 seconds. - 3. Server verifies the claim locally, forwards it to the broker and sends - `{"type":"claim_sent"}`. This does not mean the broker accepted it; a rejection - closes the socket with 4003. - 4. Client sends each broadcast as a binary frame containing a full SOC at the cohort address. + 1. Server sends `{"type":"challenge","challenge":"","broker":"","dataTopic":"","authTopic":""}`. + `dataTopic` is `keccak256(topic | challenge)`, the session feed for updates; + `authTopic` is `keccak256("bps-service:v1" | 0x02 | topic | challenge)`. + 2. Client sends binary frames `kind (1) | index (8, big-endian) | soc`, where `kind` is + 1 (DATA) or 2 (AUTH) and `soc` is a single-owner chunk signed by `owner` with + `id = keccak256(sessionTopic | index)`. An AUTH chunk is empty and only authenticates + the stream. The first frame must arrive within 30 seconds. Indexes must not be reused + across sessions; frames below the broker's cursor are dropped silently. - Close codes: 4001 invalid claim, 4002 invalid message (bad SOC, wrong frame type, - claim timeout), 4003 broker stream closed. + Close codes: 4001 invalid soc, 4002 invalid message (wrong frame type, unknown kind, + auth timeout), 4003 broker stream closed. tags: - BPS parameters: @@ -1134,12 +1145,6 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" required: true description: Overlay address of the broker node. Must not be this node. - - in: query - name: identity - schema: - $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" - required: true - description: Ethereum address identifying the joiner. responses: "200": description: Establishes a WebSocket subscription for publishing broadcasts diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index d4d3fe71e9d..ba9fdf3630d 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -29,6 +29,7 @@ import ( mockac "github.com/ethersphere/bee/v2/pkg/accesscontrol/mock" accountingmock "github.com/ethersphere/bee/v2/pkg/accounting/mock" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/pipeline" @@ -96,7 +97,7 @@ type testServerOptions struct { Resolver resolver.Interface Pss pss.Interface Gsoc gsoc.Listener - Bps api.BPSService + Bps bps.Interface WsPath string WsPingPeriod time.Duration Logger log.Logger diff --git a/pkg/api/bps.go b/pkg/api/bps.go index f802cb2bed3..a846ec3bcde 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -6,6 +6,7 @@ package api import ( "context" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -15,69 +16,43 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/bps" - "github.com/ethersphere/bee/v2/pkg/cac" - "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/jsonhttp" "github.com/ethersphere/bee/v2/pkg/log" - "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/gorilla/mux" "github.com/gorilla/websocket" ) const ( - bpsCloseInvalidClaim = 4001 // claim failed local verification or was malformed + bpsCloseInvalidSOC = 4001 // a chunk that does not validate under the session challenge bpsCloseInvalidMessage = 4002 // frame violates the endpoint protocol bpsCloseBrokerGone = 4003 // the p2p stream to the broker ended - bpsMaxCloseReason = 123 // websocket control frame payload limit minus the code - bpsMaxFrameSize = swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + swarm.ChunkSize + bpsMaxCloseReason = 123 // websocket control frame payload limit minus the code + bpsFrameHeader = 1 + 8 // publish frame header: kind | index (big-endian) + bpsMaxSOCSize = swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + swarm.ChunkSize + bpsMaxFrameSize = bpsFrameHeader + bpsMaxSOCSize ) -// bpsTopicPrefix domain-separates the cohort SOC id from the owner's other SOCs. -var bpsTopicPrefix = []byte("bps-claim") - -// bpsClaimTimeout bounds how long a publisher may take to send its claim. -var bpsClaimTimeout = 30 * time.Second +// bpsAuthTimeout bounds how long a publisher may take to send its first frame. +var bpsAuthTimeout = 30 * time.Second var errBPSBrokerGone = errors.New("broker stream closed") type bpsRequest struct { owner common.Address - id []byte topic []byte - addr swarm.Address broker swarm.Address identity []byte + cursor uint64 } type bpsChallengeMessage struct { Type string `json:"type"` Challenge string `json:"challenge"` Broker string `json:"broker"` - ID string `json:"id"` -} - -type bpsClaimMessage struct { - Type string `json:"type"` - Signature string `json:"signature"` -} - -type bpsClaimSentMessage struct { - Type string `json:"type"` -} - -// bpsTopicAddress derives the cohort SOC id and address for owner and topic. -func bpsTopicAddress(owner common.Address, topic []byte) ([]byte, swarm.Address, error) { - id, err := crypto.LegacyKeccak256(append(append([]byte{}, bpsTopicPrefix...), topic...)) - if err != nil { - return nil, swarm.ZeroAddress, err - } - addr, err := soc.CreateAddress(id, owner.Bytes()) - if err != nil { - return nil, swarm.ZeroAddress, err - } - return id, addr, nil + DataTopic string `json:"dataTopic"` + AuthTopic string `json:"authTopic"` } func (s *Service) bpsSubscribeWsHandler(w http.ResponseWriter, r *http.Request) { @@ -86,6 +61,10 @@ func (s *Service) bpsSubscribeWsHandler(w http.ResponseWriter, r *http.Request) if !ok { return } + if len(req.identity) == 0 { + jsonhttp.BadRequest(w, "missing identity") + return + } conn, ok := s.bpsUpgrade(w, r, logger) if !ok { return @@ -127,7 +106,8 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger queries := struct { Broker swarm.Address `map:"broker"` - Identity []byte `map:"identity" validate:"required,len=20"` + Identity []byte `map:"identity" validate:"omitempty,len=20"` + Cursor uint64 `map:"cursor"` }{} if response := s.mapStructure(r.URL.Query(), &queries); response != nil { response("invalid query params", logger, w) @@ -146,19 +126,12 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger return bpsRequest{}, false } - id, addr, err := bpsTopicAddress(paths.Owner, paths.Topic) - if err != nil { - logger.Debug("derive topic address failed", "error", err) - jsonhttp.InternalServerError(w, "derive topic address failed") - return bpsRequest{}, false - } return bpsRequest{ owner: paths.Owner, - id: id, topic: paths.Topic, - addr: addr, broker: queries.Broker, identity: queries.Identity, + cursor: queries.Cursor, }, true } @@ -258,11 +231,10 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo }() sess, err := s.bps.Join(ctx, bps.JoinRequest{ - Broker: req.broker, - Binding: bps.BindingFeed, - Topic: req.topic, - Principal: req.owner.Bytes(), - Identity: req.identity, + Broker: req.broker, + Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, + Identity: req.identity, + Cursor: req.cursor, }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -275,11 +247,16 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo for { select { - case data, ok := <-sess.Messages(): + case msg, ok := <-sess.Messages(): if !ok { s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) return } + // challenge | index | soc: what the client needs to re-verify the chunk + data := make([]byte, 0, bps.ChallengeSize+8+len(msg.SOC)) + data = append(data, msg.Challenge...) + data = binary.BigEndian.AppendUint64(data, msg.Index) + data = append(data, msg.SOC...) if err := s.bpsWrite(conn, websocket.BinaryMessage, data); err != nil { logger.Debug("write broadcast failed", "error", err) return @@ -303,43 +280,25 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo } } -// bpsVerifyClaim checks a claim frame the same way the broker will and -// returns the claim SOC bytes, or a close code and reason on failure. -func bpsVerifyClaim(f bpsFrame, req bpsRequest, challenge []byte) ([]byte, int, string) { - if f.typ != websocket.TextMessage { - return nil, bpsCloseInvalidMessage, "expected claim" +// bpsParseFrame checks a publish frame kind | index | soc the same way the +// broker will, or returns a close code and reason on failure. +func bpsParseFrame(f bpsFrame, req bpsRequest, challenge []byte) (bps.Kind, uint64, []byte, int, string) { + if f.typ != websocket.BinaryMessage { + return 0, 0, nil, bpsCloseInvalidMessage, "expected binary frame" } - var m bpsClaimMessage - if err := json.Unmarshal(f.data, &m); err != nil { - return nil, bpsCloseInvalidClaim, "malformed claim" + if len(f.data) < bpsFrameHeader { + return 0, 0, nil, bpsCloseInvalidMessage, "short frame" } - if m.Type != "claim" { - return nil, bpsCloseInvalidMessage, "expected claim" + kind := bps.Kind(f.data[0]) + if kind != bps.KindData && kind != bps.KindAuth { + return 0, 0, nil, bpsCloseInvalidMessage, "unknown kind" } - sig, err := hex.DecodeString(m.Signature) - if err != nil || len(sig) != swarm.SocSignatureSize { - return nil, bpsCloseInvalidClaim, "invalid signature" + index := binary.BigEndian.Uint64(f.data[1:bpsFrameHeader]) + chunk := f.data[bpsFrameHeader:] + if err := bps.Verify(kind, challenge, index, chunk, req.topic, req.owner.Bytes()); err != nil { + return 0, 0, nil, bpsCloseInvalidSOC, "invalid soc" } - payload := make([]byte, 0, len(challenge)+swarm.HashSize) - payload = append(append(payload, challenge...), req.broker.Bytes()...) - ch, err := cac.New(payload) - if err != nil { - return nil, bpsCloseInvalidClaim, "invalid claim payload" - } - sc, err := soc.NewSigned(req.id, ch, req.owner.Bytes(), sig) - if err != nil { - return nil, bpsCloseInvalidClaim, "invalid claim" - } - chunk, err := sc.Chunk() - if err != nil { - return nil, bpsCloseInvalidClaim, "invalid claim" - } - // NewSigned does not verify the signature; Valid recovers the signer - // and checks that it maps to the cohort address. - if !chunk.Address().Equal(req.addr) || !soc.Valid(chunk) { - return nil, bpsCloseInvalidClaim, "claim verification failed" - } - return chunk.Data(), 0, "" + return kind, index, chunk, 0, "" } func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log.Logger) { @@ -362,12 +321,11 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. _ = conn.Close() }() + // the publisher declares the principal as its identity and authenticates with its first frame sess, err := s.bps.Join(ctx, bps.JoinRequest{ - Broker: req.broker, - Binding: bps.BindingFeed, - Topic: req.topic, - Principal: req.owner.Bytes(), - Identity: req.identity, + Broker: req.broker, + Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, + Identity: req.owner.Bytes(), }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -377,63 +335,57 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. defer sess.Close() challenge := sess.Challenge() - if len(challenge) != swarm.HashSize { + if len(challenge) != bps.ChallengeSize { logger.Debug("invalid challenge length", "broker", req.broker, "length", len(challenge)) s.bpsClose(conn, bpsCloseBrokerGone, "invalid challenge") return } + dataTopic, err := bps.SessionTopic(bps.KindData, req.topic, challenge) + if err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, "session topic") + return + } + authTopic, err := bps.SessionTopic(bps.KindAuth, req.topic, challenge) + if err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, "session topic") + return + } if err := s.bpsWriteJSON(conn, bpsChallengeMessage{ Type: "challenge", Challenge: hex.EncodeToString(challenge), Broker: req.broker.String(), - ID: hex.EncodeToString(req.id), + DataTopic: hex.EncodeToString(dataTopic), + AuthTopic: hex.EncodeToString(authTopic), }); err != nil { logger.Debug("write challenge failed", "error", err) return } - claimTimer := time.NewTimer(bpsClaimTimeout) - defer claimTimer.Stop() + authTimer := time.NewTimer(bpsAuthTimeout) + defer authTimer.Stop() frames, gone := bpsReadFrames(conn, quit) - claimed := false + authenticated := false for { select { case f := <-frames: - if !claimed { - claim, code, reason := bpsVerifyClaim(f, req, challenge) - if code != 0 { - s.bpsClose(conn, code, reason) - return - } - if err := sess.Claim(ctx, claim); err != nil { - s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) - return - } - claimed = true - claimTimer.Stop() - if err := s.bpsWriteJSON(conn, bpsClaimSentMessage{Type: "claim_sent"}); err != nil { - logger.Debug("write claim_sent failed", "error", err) - return - } - continue - } - if f.typ != websocket.BinaryMessage { - s.bpsClose(conn, bpsCloseInvalidMessage, "expected binary soc") - return - } - if !soc.Valid(swarm.NewChunk(req.addr, f.data)) { - s.bpsClose(conn, bpsCloseInvalidMessage, "invalid soc") + kind, index, chunk, code, reason := bpsParseFrame(f, req, challenge) + if code != 0 { + s.bpsClose(conn, code, reason) return } - if err := sess.Publish(ctx, f.data); err != nil { + if err := sess.Publish(ctx, kind, index, chunk); err != nil { s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) return } - case <-claimTimer.C: - if !claimed { - s.bpsClose(conn, bpsCloseInvalidMessage, "claim timeout") + if !authenticated { + authenticated = true + authTimer.Stop() + } + case <-authTimer.C: + if !authenticated { + s.bpsClose(conn, bpsCloseInvalidMessage, "auth timeout") return } case <-sess.Done(): diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index f21d6191c71..d316c0cb490 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -7,6 +7,7 @@ package api_test import ( "bytes" "context" + "encoding/binary" "encoding/hex" "errors" "fmt" @@ -17,6 +18,7 @@ import ( "time" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/jsonhttp" @@ -31,14 +33,13 @@ type fakeBPS struct { sess *fakeSession joinErr error - mu sync.Mutex - broker swarm.Address - addr swarm.Address + mu sync.Mutex + req bps.JoinRequest } -func (f *fakeBPS) Join(_ context.Context, broker, addr swarm.Address) (api.BPSSession, error) { +func (f *fakeBPS) Join(_ context.Context, req bps.JoinRequest) (bps.Session, error) { f.mu.Lock() - f.broker, f.addr = broker, addr + f.req = req f.mu.Unlock() if f.joinErr != nil { return nil, f.joinErr @@ -46,15 +47,19 @@ func (f *fakeBPS) Join(_ context.Context, broker, addr swarm.Address) (api.BPSSe return f.sess, nil } +type published struct { + kind bps.Kind + index uint64 + soc []byte +} + type fakeSession struct { challenge []byte - msgs chan []byte + msgs chan bps.Message done chan struct{} - claims chan []byte - published chan []byte + published chan published closed chan struct{} - claimErr error publishErr error doneOnce sync.Once @@ -67,32 +72,24 @@ func newFakeSession() *fakeSession { copy(challenge, "bps-test-challenge") return &fakeSession{ challenge: challenge, - msgs: make(chan []byte), + msgs: make(chan bps.Message), done: make(chan struct{}), - claims: make(chan []byte, 1), - published: make(chan []byte, 8), + published: make(chan published, 8), closed: make(chan struct{}), } } -func (f *fakeSession) Challenge() []byte { return f.challenge } -func (f *fakeSession) Messages() <-chan []byte { return f.msgs } -func (f *fakeSession) Done() <-chan struct{} { return f.done } -func (f *fakeSession) Err() error { return f.err } - -func (f *fakeSession) Claim(_ context.Context, b []byte) error { - if f.claimErr != nil { - return f.claimErr - } - f.claims <- b - return nil -} +func (f *fakeSession) Challenge() []byte { return f.challenge } +func (f *fakeSession) Messages() <-chan bps.Message { return f.msgs } +func (f *fakeSession) Cursor() uint64 { return 0 } +func (f *fakeSession) Done() <-chan struct{} { return f.done } +func (f *fakeSession) Err() error { return f.err } -func (f *fakeSession) Publish(_ context.Context, b []byte) error { +func (f *fakeSession) Publish(_ context.Context, kind bps.Kind, index uint64, b []byte) error { if f.publishErr != nil { return f.publishErr } - f.published <- b + f.published <- published{kind: kind, index: index, soc: b} return nil } @@ -116,9 +113,8 @@ type bpsFixture struct { sess *fakeSession signer crypto.Signer owner []byte + identity []byte topic []byte - id []byte - addr swarm.Address broker swarm.Address self swarm.Address } @@ -136,25 +132,18 @@ func newBPSFixture(t *testing.T) *bpsFixture { } topic := make([]byte, 32) copy(topic, "bps-test-topic") - id, err := crypto.LegacyKeccak256(append([]byte("bps-claim"), topic...)) - if err != nil { - t.Fatal(err) - } - addr, err := soc.CreateAddress(id, owner.Bytes()) - if err != nil { - t.Fatal(err) - } + identity := make([]byte, 20) + copy(identity, "bps-test-subscriber") sess := newFakeSession() f := &bpsFixture{ - bps: &fakeBPS{sess: sess}, - sess: sess, - signer: signer, - owner: owner.Bytes(), - topic: topic, - id: id, - addr: addr, - broker: swarm.RandAddress(t), - self: swarm.RandAddress(t), + bps: &fakeBPS{sess: sess}, + sess: sess, + signer: signer, + owner: owner.Bytes(), + identity: identity, + topic: topic, + broker: swarm.RandAddress(t), + self: swarm.RandAddress(t), } f.client, _, f.listener, _, _ = newTestServer(t, testServerOptions{ Bps: f.bps, @@ -170,7 +159,8 @@ func (f *bpsFixture) path(endpoint string) string { func (f *bpsFixture) dial(t *testing.T, endpoint string) *websocket.Conn { t.Helper() - u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path(endpoint), RawQuery: "broker=" + f.broker.String()} + q := "broker=" + f.broker.String() + "&identity=" + hex.EncodeToString(f.identity) + u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path(endpoint), RawQuery: q} conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) if err != nil { t.Fatalf("dial %s: %v", u.String(), err) @@ -182,26 +172,30 @@ func (f *bpsFixture) dial(t *testing.T, endpoint string) *websocket.Conn { return conn } -// signedSOC returns full SOC bytes at f.addr wrapping payload. -func (f *bpsFixture) signedSOC(t *testing.T, signer crypto.Signer, payload []byte) []byte { +// signedSOC returns the SOC bytes of the session feed update of kind at index, +// signed by signer under challenge. +func (f *bpsFixture) signedSOC(t *testing.T, signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { t.Helper() + id, err := bps.ID(kind, f.topic, challenge, index) + if err != nil { + t.Fatal(err) + } ch, err := cac.New(payload) if err != nil { t.Fatal(err) } - c, err := soc.New(f.id, ch).Sign(signer) + c, err := soc.New(id, ch).Sign(signer) if err != nil { t.Fatal(err) } return c.Data() } -// claimSignature signs the claim payload challenge|broker with signer. -func (f *bpsFixture) claimSignature(t *testing.T, signer crypto.Signer, challenge []byte) string { - t.Helper() - payload := append(append([]byte{}, challenge...), f.broker.Bytes()...) - data := f.signedSOC(t, signer, payload) - return hex.EncodeToString(data[swarm.HashSize : swarm.HashSize+swarm.SocSignatureSize]) +// frame returns a publish frame kind | index | soc. +func frame(kind bps.Kind, index uint64, chunk []byte) []byte { + b := []byte{byte(kind)} + b = binary.BigEndian.AppendUint64(b, index) + return append(b, chunk...) } func expectClose(t *testing.T, conn *websocket.Conn, code int) { @@ -263,43 +257,37 @@ func TestBPSPreUpgrade(t *testing.T) { func TestBPSSubscribe(t *testing.T) { t.Parallel() - t.Run("forwards valid soc", func(t *testing.T) { + t.Run("forwards delivery", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) conn := f.dial(t, "subscribe") - msg := f.signedSOC(t, f.signer, []byte("hello cohort")) - f.sess.msgs <- msg + chunk := f.signedSOC(t, f.signer, bps.KindData, f.sess.challenge, 7, []byte("hello cohort")) + f.sess.msgs <- bps.Message{SOC: chunk, Challenge: f.sess.challenge, Index: 7} typ, got, err := conn.ReadMessage() if err != nil { t.Fatal(err) } - if typ != websocket.BinaryMessage || !bytes.Equal(got, msg) { - t.Fatalf("got type %d %x, want binary %x", typ, got, msg) + want := append(append([]byte{}, f.sess.challenge...), 0, 0, 0, 0, 0, 0, 0, 7) + want = append(want, chunk...) + if typ != websocket.BinaryMessage || !bytes.Equal(got, want) { + t.Fatalf("got type %d %x, want binary %x", typ, got, want) } f.bps.mu.Lock() - gotAddr, gotBroker := f.bps.addr, f.bps.broker + req := f.bps.req f.bps.mu.Unlock() - if !gotAddr.Equal(f.addr) || !gotBroker.Equal(f.broker) { - t.Fatalf("join got addr %s broker %s, want %s %s", gotAddr, gotBroker, f.addr, f.broker) + if !req.Broker.Equal(f.broker) || !bytes.Equal(req.Spec.Principal, f.owner) || !bytes.Equal(req.Identity, f.identity) || !bytes.Equal(req.Spec.Topic, f.topic) { + t.Fatalf("unexpected join request %+v", req) } }) - t.Run("drops invalid soc", func(t *testing.T) { + t.Run("missing identity", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := f.dial(t, "subscribe") - other, _ := crypto.GenerateSecp256k1Key() - f.sess.msgs <- f.signedSOC(t, crypto.NewDefaultSigner(other), []byte("forged")) - f.sess.msgs <- []byte("garbage") - valid := f.signedSOC(t, f.signer, []byte("real")) - f.sess.msgs <- valid - _, got, err := conn.ReadMessage() - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(got, valid) { - t.Fatalf("got %x, want only the valid soc %x", got, valid) - } + jsonhttptest.Request(t, f.client, http.MethodGet, f.path("subscribe")+"?broker="+f.broker.String(), http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "missing identity", + }), + ) }) t.Run("client frame closes 4002", func(t *testing.T) { @@ -357,12 +345,16 @@ func readChallenge(t *testing.T, f *bpsFixture, conn *websocket.Conn) []byte { Type string `json:"type"` Challenge string `json:"challenge"` Broker string `json:"broker"` - ID string `json:"id"` + DataTopic string `json:"dataTopic"` + AuthTopic string `json:"authTopic"` } if err := conn.ReadJSON(&m); err != nil { t.Fatal(err) } - if m.Type != "challenge" || m.Broker != f.broker.String() || m.ID != hex.EncodeToString(f.id) { + dataTopic, _ := bps.SessionTopic(bps.KindData, f.topic, f.sess.challenge) + authTopic, _ := bps.SessionTopic(bps.KindAuth, f.topic, f.sess.challenge) + if m.Type != "challenge" || m.Broker != f.broker.String() || + m.DataTopic != hex.EncodeToString(dataTopic) || m.AuthTopic != hex.EncodeToString(authTopic) { t.Fatalf("unexpected challenge message %+v", m) } challenge, err := hex.DecodeString(m.Challenge) @@ -372,69 +364,45 @@ func readChallenge(t *testing.T, f *bpsFixture, conn *websocket.Conn) []byte { return challenge } -func sendClaim(t *testing.T, conn *websocket.Conn, sig string) { - t.Helper() - if err := conn.WriteJSON(map[string]string{"type": "claim", "signature": sig}); err != nil { - t.Fatal(err) - } -} - -func expectClaimSent(t *testing.T, conn *websocket.Conn) { +func expectPublished(t *testing.T, f *bpsFixture, kind bps.Kind, index uint64, chunk []byte) { t.Helper() - var m struct { - Type string `json:"type"` - } - if err := conn.ReadJSON(&m); err != nil { - t.Fatal(err) - } - if m.Type != "claim_sent" { - t.Fatalf("got %q, want claim_sent", m.Type) + select { + case got := <-f.sess.published: + if got.kind != kind || got.index != index || !bytes.Equal(got.soc, chunk) { + t.Fatalf("published %d/%d %x, want %d/%d %x", got.kind, got.index, got.soc, kind, index, chunk) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for publish") } } -// claimed dials publish and completes the claim handshake. -func claimed(t *testing.T, f *bpsFixture) *websocket.Conn { - t.Helper() - conn := f.dial(t, "publish") - challenge := readChallenge(t, f, conn) - sendClaim(t, conn, f.claimSignature(t, f.signer, challenge)) - expectClaimSent(t, conn) - return conn -} - func TestBPSPublish(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := claimed(t, f) + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) - claim := <-f.sess.claims - if !soc.Valid(swarm.NewChunk(f.addr, claim)) { - t.Fatal("claim passed to session is not a valid soc at the topic address") + f.bps.mu.Lock() + req := f.bps.req + f.bps.mu.Unlock() + if !bytes.Equal(req.Identity, f.owner) { + t.Fatalf("publisher joined as %x, want the principal %x", req.Identity, f.owner) } - sc, err := soc.FromChunk(swarm.NewChunk(f.addr, claim)) - if err != nil { + + auth := f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { t.Fatal(err) } - wantPayload := append(append([]byte{}, f.sess.challenge...), f.broker.Bytes()...) - if got := sc.WrappedChunk().Data()[swarm.SpanSize:]; !bytes.Equal(got, wantPayload) { - t.Fatalf("claim payload %x, want %x", got, wantPayload) - } + expectPublished(t, f, bps.KindAuth, 0, auth) - msg := f.signedSOC(t, f.signer, []byte("broadcast")) - if err := conn.WriteMessage(websocket.BinaryMessage, msg); err != nil { + data := f.signedSOC(t, f.signer, bps.KindData, challenge, 3, []byte("broadcast")) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 3, data)); err != nil { t.Fatal(err) } - select { - case got := <-f.sess.published: - if !bytes.Equal(got, msg) { - t.Fatalf("published %x, want %x", got, msg) - } - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for publish") - } + expectPublished(t, f, bps.KindData, 3, data) } -func TestBPSPublishClaimErrors(t *testing.T) { +func TestBPSPublishFrameErrors(t *testing.T) { t.Parallel() other, _ := crypto.GenerateSecp256k1Key() @@ -448,49 +416,64 @@ func TestBPSPublishClaimErrors(t *testing.T) { { name: "wrong signer", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { - sendClaim(t, conn, f.claimSignature(t, otherSigner, challenge)) + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, otherSigner, bps.KindData, challenge, 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { name: "wrong challenge", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { - sendClaim(t, conn, f.claimSignature(t, f.signer, make([]byte, 32))) + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, make([]byte, 32), 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "malformed json", - send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { - _ = conn.WriteMessage(websocket.TextMessage, []byte("{not json")) + name: "index not the signed one", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 1, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "short signature", - send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { - sendClaim(t, conn, "abcd") + name: "auth relabelled as data", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "0x-prefixed signature", + name: "auth with payload", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { - sendClaim(t, conn, "0x"+f.claimSignature(t, f.signer, challenge)) + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "wrong message type", + name: "unknown kind", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.Kind(9), 0, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) + }, + code: api.BPSCloseInvalidMessage, + }, + { + name: "short frame", send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { - _ = conn.WriteJSON(map[string]string{"type": "hello"}) + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, []byte{1, 2}) }, code: api.BPSCloseInvalidMessage, }, { - name: "binary before claim", - send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { - _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, f.signer, []byte("early"))) + name: "text frame", + send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { + t.Helper() + _ = conn.WriteJSON(map[string]string{"type": "auth"}) }, code: api.BPSCloseInvalidMessage, }, @@ -503,66 +486,38 @@ func TestBPSPublishClaimErrors(t *testing.T) { tc.send(t, f, conn, challenge) expectClose(t, conn, tc.code) waitSessionClosed(t, f) - if len(f.sess.claims) != 0 { - t.Fatal("rejected claim reached the session") + if len(f.sess.published) != 0 { + t.Fatal("rejected frame reached the session") } }) } } -func TestBPSPublishAfterClaimErrors(t *testing.T) { +func TestBPSPublishSessionErrors(t *testing.T) { t.Parallel() - t.Run("invalid soc closes 4002", func(t *testing.T) { - t.Parallel() - f := newBPSFixture(t) - conn := claimed(t, f) - other, _ := crypto.GenerateSecp256k1Key() - _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, crypto.NewDefaultSigner(other), []byte("forged"))) - expectClose(t, conn, api.BPSCloseInvalidMessage) - if len(f.sess.published) != 0 { - t.Fatal("invalid soc was published") - } - }) - - t.Run("text frame closes 4002", func(t *testing.T) { - t.Parallel() - f := newBPSFixture(t) - conn := claimed(t, f) - _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"type":"claim"}`)) - expectClose(t, conn, api.BPSCloseInvalidMessage) - }) - t.Run("broker gone closes 4003", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := claimed(t, f) + conn := f.dial(t, "publish") + readChallenge(t, f, conn) f.sess.end(errors.New("stream reset")) expectClose(t, conn, api.BPSCloseBrokerGone) waitSessionClosed(t, f) }) - t.Run("claim write error closes 4003", func(t *testing.T) { - t.Parallel() - f := newBPSFixture(t) - f.sess.claimErr = errors.New("write failed") - conn := f.dial(t, "publish") - challenge := readChallenge(t, f, conn) - sendClaim(t, conn, f.claimSignature(t, f.signer, challenge)) - expectClose(t, conn, api.BPSCloseBrokerGone) - }) - t.Run("publish error closes 4003", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) f.sess.publishErr = errors.New("write failed") - conn := claimed(t, f) - _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, f.signer, []byte("x"))) + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) expectClose(t, conn, api.BPSCloseBrokerGone) }) } -func TestBPSPublishClientGoneBeforeClaim(t *testing.T) { +func TestBPSPublishClientGoneBeforeAuth(t *testing.T) { t.Parallel() f := newBPSFixture(t) conn := f.dial(t, "publish") @@ -570,25 +525,22 @@ func TestBPSPublishClientGoneBeforeClaim(t *testing.T) { // Send a close frame rather than Close, which the dial cleanup would // report as an error on the already closed connection. _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) - select { - case <-f.sess.closed: - case <-time.After(5 * time.Second): - t.Fatal("session not closed after client went away") - } + waitSessionClosed(t, f) } func TestBPSPublishOversizedFrame(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := claimed(t, f) + conn := f.dial(t, "publish") + readChallenge(t, f, conn) _ = conn.WriteMessage(websocket.BinaryMessage, make([]byte, 5000)) expectClose(t, conn, websocket.CloseMessageTooBig) waitSessionClosed(t, f) } -// Not parallel: mutates the package-level claim timeout. -func TestBPSPublishClaimTimeout(t *testing.T) { - restore := api.SetBPSClaimTimeout(100 * time.Millisecond) +// Not parallel: mutates the package-level auth timeout. +func TestBPSPublishAuthTimeout(t *testing.T) { + restore := api.SetBPSAuthTimeout(100 * time.Millisecond) defer restore() f := newBPSFixture(t) conn := f.dial(t, "publish") diff --git a/pkg/api/export_test.go b/pkg/api/export_test.go index 173d9c94656..7b424db67b1 100644 --- a/pkg/api/export_test.go +++ b/pkg/api/export_test.go @@ -166,15 +166,15 @@ const RootPath = rootPath func (s *Service) Router() *mux.Router { return s.router } const ( - BPSCloseInvalidClaim = bpsCloseInvalidClaim + BPSCloseInvalidSOC = bpsCloseInvalidSOC BPSCloseInvalidMessage = bpsCloseInvalidMessage BPSCloseBrokerGone = bpsCloseBrokerGone ) -// SetBPSClaimTimeout overrides the claim timeout and returns a restore func. +// SetBPSAuthTimeout overrides the auth timeout and returns a restore func. // Callers must not run in parallel with other bps tests. -func SetBPSClaimTimeout(d time.Duration) func() { - prev := bpsClaimTimeout - bpsClaimTimeout = d - return func() { bpsClaimTimeout = prev } +func SetBPSAuthTimeout(d time.Duration) func() { + prev := bpsAuthTimeout + bpsAuthTimeout = d + return func() { bpsAuthTimeout = prev } } diff --git a/pkg/api/status_test.go b/pkg/api/status_test.go index 5844e86547e..440dcf2022c 100644 --- a/pkg/api/status_test.go +++ b/pkg/api/status_test.go @@ -74,7 +74,6 @@ func TestGetStatus(t *testing.T) { jsonhttptest.WithExpectedJSONResponse(ssr), ) }) - } // TestGetStatusPeersIncludesBootnodes is a regression test for diff --git a/pkg/bps/TODO b/pkg/bps/TODO deleted file mode 100644 index c4e0b659d75..00000000000 --- a/pkg/bps/TODO +++ /dev/null @@ -1,52 +0,0 @@ -● Here's what's still open, from the earlier reviews plus a few smaller things I noticed while working in the code. - - Protocol and security - - 1. Publisher tracking (#7): add cohort.publisher and allow only one publisher per cohort. On takeover, close the old publisher's stream. Doing - the check, the swap and the eviction under one lock also stops two claims from racing each other. - 2. Owner checks (#8): - - Store the owner recovered from the claim in the cohort. - - The broker should check that each broadcast is a valid SOC signed by that owner. - - Subscribers should check too, which means deciding how they learn the owner. - 3. Members keyed by stream, not overlay (#9): a second join from the same peer currently overwrites the first, and either stream's left then - removes the other. Fixing this also closes the case where the publisher stalls sending on ch1 after the fan-out goroutine has exited. - 4. Domain-separated topic ID: use id = keccak("bps-claim" | t) so the claim chunk can't overwrite the owner's own SOC or GSOC in Swarm. This is a - client-side convention; a helper function would enforce it. - 5. Limits on cohort creation: reject Join.Topic values that aren't 32 bytes, and cap cohorts per peer and in total. - 6. Replay and ordering of broadcasts: lastSeen is commented out, so nothing stops a broker from re-sending old messages. - - Client (Join) - - 7. The first message on tx is always sent as the claim. Give the claim its own path or a state flag. - 8. Wrong log messages: several write errors are logged as "read join ack". - - Code quality - - 9. newMember panics if rand.Read fails; it should return an error. - 10. The ad-hoc errors.New messages should be bps:-prefixed sentinel errors. - 11. soc.FromChunk's error is ignored. It can't fail once Valid passes, but it deserves a comment. - 12. The fan-out goroutine takes the service-wide s.mtx for every message. A lock per cohort will be needed once there are many cohorts. - 13. Stale comments: - - The package doc still describes a "hello-world wire protocol used to greet other peers". - - The handler comment still says the "next challenge changes randomly", but nonces are now per member. - - Integration - - 14. Wiring: nothing in pkg/node/node.go calls bps.New yet. It needs swarmAddress and o.FullNodeMode, and the protocol has to be registered with - p2p. - - Tests - - 15. Claim cases not covered yet: - - a claim signed for a different broker - - a stale nonce, e.g. from an earlier stream - - a bad signature, or a signer other than the owner - - publisher takeover, once #1 is done - - a subscriber leaving and being removed - - an unverified broadcast being rejected, once #2 is done - - Housekeeping - - 16. Unrelated diff: regenerated .pb.go files in 12 unrelated packages are still modified in the working tree. They're safe to revert. - - I'd do 3, then 1 and 2 together (tracking the publisher and storing the owner are really one change), then 7, then 14. diff --git a/pkg/bps/binding.go b/pkg/bps/binding.go deleted file mode 100644 index f2482bda946..00000000000 --- a/pkg/bps/binding.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2026 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package bps - -import ( - "bytes" - "errors" - "fmt" - - "github.com/ethersphere/bee/v2/pkg/bps/pb" - "github.com/ethersphere/bee/v2/pkg/crypto" - "github.com/ethersphere/bee/v2/pkg/soc" - "github.com/ethersphere/bee/v2/pkg/swarm" -) - -// claimIDPrefix domain-separates the cohort's claim SOC id from the -// principal's other single owner chunks. -var claimIDPrefix = []byte("bps-claim") - -// errUnsupportedBinding is returned for a Join naming a topic binding this -// node does not implement. -var errUnsupportedBinding = errors.New("bps: unsupported topic binding") - -// errClaimNotQualified is returned when a claim SOC does not satisfy its -// cohort's topic binding. -var errClaimNotQualified = errors.New("bps: claim does not qualify") - -// binding decides how a cohort's publisher claim is verified. The topic -// binding named in Join picks the implementation; feed is the only one for now. -type binding interface { - // claimAddress returns the SOC address a valid claim must have for the - // cohort's topic and governing principal. - claimAddress(topic, principal []byte) (swarm.Address, error) - // verifyClaim checks the decoded claim's owner and signed payload. It - // returns nil when the claim promotes the stream to publisher. - verifyClaim(s *soc.SOC, principal, challenge, broker, topic []byte) error -} - -// bindingFor returns the binding rules for b, or errUnsupportedBinding. -func bindingFor(b pb.TopicBinding) (binding, error) { - switch b { - case pb.TopicBinding_BINDING_FEED: - return feedBinding{}, nil - } - return nil, fmt.Errorf("%s: %w", b, errUnsupportedBinding) -} - -// feedBinding verifies the publisher claim for a feed topic. The claim is a -// SOC with id keccak256("bps-claim" | topic), owned by the governing principal, -// whose wrapped payload signs the broker overlay and feed topic together with -// the per-member challenge. -type feedBinding struct{} - -func (feedBinding) claimAddress(topic, principal []byte) (swarm.Address, error) { - id, err := feedClaimID(topic) - if err != nil { - return swarm.ZeroAddress, err - } - return soc.CreateAddress(id, principal) -} - -func (feedBinding) verifyClaim(s *soc.SOC, principal, challenge, broker, topic []byte) error { - if !bytes.Equal(s.OwnerAddress(), principal) { - return fmt.Errorf("claim owner is not the principal: %w", errClaimNotQualified) - } - innerProof := make([]byte, 0, len(challenge)+len(broker)+len(topic)) - innerProof = append(innerProof, challenge...) - innerProof = append(innerProof, broker...) - innerProof = append(innerProof, topic...) - if !bytes.Equal(s.WrappedChunk().Data()[swarm.SpanSize:], innerProof) { - return fmt.Errorf("claim payload mismatch: %w", errClaimNotQualified) - } - return nil -} - -// feedClaimID derives the claim SOC id for a feed topic. -func feedClaimID(topic []byte) ([]byte, error) { - return crypto.LegacyKeccak256(append(append([]byte{}, claimIDPrefix...), topic...)) -} diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 53756ade28b..bbf45feb72b 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -2,8 +2,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package bps exposes a simple hello-world wire protocol -// used to greet other peers. +// Package bps implements BPS-lite (SWIP-74): a single-publisher live stream +// over a feed, through one broker, one hop. +// +// A cohort is identified by its spec {topic, FEED_TOPIC, principal}. Every +// peer joins with the spec and its own identity and is answered with a random +// per-stream challenge. The principal publishes ordinary single-owner chunks that +// are updates of the session feed on keccak256(topic | challenge); its first +// valid frame authenticates the stream. The broker keeps one cursor per cohort and +// delivers every accepted update to every subscriber stream. package bps import ( @@ -13,13 +20,14 @@ import ( "errors" "fmt" "sync" + "sync/atomic" + "time" "github.com/ethersphere/bee/v2/pkg/bps/pb" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" - "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" ) @@ -32,38 +40,103 @@ const ( streamName = "bps" ) -// broadcastBufferSize is the per-member buffer for broadcast messages. -// Delivery is lossy by design (a slow member must not stall the cohort), -// so the buffer absorbs short bursts while the member's stream is busy. -const broadcastBufferSize = 1 - var ( - errNotBroker = errors.New("not a broker") - errJoinRejected = errors.New("bps: join rejected") - errSessionClosed = errors.New("bps: session closed") + errNotBroker = errors.New("bps: not a broker") + errJoinRefused = errors.New("bps: join refused") + errInvalidJoin = errors.New("bps: invalid join") + errSessionClosed = errors.New("bps: session closed") + errProtocolError = errors.New("bps: protocol error") + errViolation = errors.New("bps: protocol violation") + errWrongStream = errors.New("bps: frame from subscriber stream") + errCohortReclaimed = errors.New("bps: cohort reclaimed") + errFull = errors.New("bps: capacity bound") + errNotPublisher = errors.New("bps: session is not a publisher") + errInvalidDelivery = errors.New("bps: invalid delivery") ) -// Service is the bps protocol service. +// Options are the broker's resource bounds. Zero values take the defaults. +type Options struct { + MaxSubscribers int // subscriber streams per cohort + MaxCohorts int // live cohorts per broker + MaxCohortsPerPeer int // cohorts per peer connection + MaxStreamsPerPeerCohort int // streams per peer connection per cohort + InactivityTimeout time.Duration // reclaims a cohort with no accepted frame + AuthTimeout time.Duration // disconnects a pending stream that has not authenticated + QueueSize int // outbound frames per subscriber stream + ViolationBlocklist time.Duration // blocklist duration for a protocol violation + BlocklistDuration time.Duration // blocklist duration for an auth timeout +} + +// DefaultOptions are the SWIP-74 recommended bounds. +var DefaultOptions = Options{ + MaxSubscribers: 1024, + MaxCohorts: 512, + MaxCohortsPerPeer: 16, + MaxStreamsPerPeerCohort: 2, + InactivityTimeout: 10 * time.Minute, + AuthTimeout: 30 * time.Second, + QueueSize: 64, + ViolationBlocklist: 10 * time.Minute, + BlocklistDuration: time.Minute, +} + +func (o Options) withDefaults() Options { + d := DefaultOptions + if o.MaxSubscribers > 0 { + d.MaxSubscribers = o.MaxSubscribers + } + if o.MaxCohorts > 0 { + d.MaxCohorts = o.MaxCohorts + } + if o.MaxCohortsPerPeer > 0 { + d.MaxCohortsPerPeer = o.MaxCohortsPerPeer + } + if o.MaxStreamsPerPeerCohort > 0 { + d.MaxStreamsPerPeerCohort = o.MaxStreamsPerPeerCohort + } + if o.InactivityTimeout > 0 { + d.InactivityTimeout = o.InactivityTimeout + } + if o.AuthTimeout > 0 { + d.AuthTimeout = o.AuthTimeout + } + if o.QueueSize > 0 { + d.QueueSize = o.QueueSize + } + if o.ViolationBlocklist > 0 { + d.ViolationBlocklist = o.ViolationBlocklist + } + if o.BlocklistDuration > 0 { + d.BlocklistDuration = o.BlocklistDuration + } + return d +} + +// Service is the bps protocol service. On a full node it is also the broker. type Service struct { fullNode bool - mtx sync.Mutex - selfOverlay swarm.Address // this node's overlay. used to verify challenges streamer p2p.Streamer - cohorts map[string]*cohort - // cohort registry - keep track of members, publisher, channels, challenge, streams? - // this is only relevant for the actual broker. subscribers don't care of manage this state at all. - // registry *CohortRegistry - - logger log.Logger + blocklister p2p.Blocklister + opts Options + metrics metrics + logger log.Logger + + mtx sync.Mutex + cohorts map[string]*cohort // by canonical spec + peers map[string]map[string]int // peer overlay -> cohort key -> streams + closed bool } -// New returns a new bps Service. -func New(streamer p2p.Streamer, overlay swarm.Address, fullNode bool, logger log.Logger) *Service { +// New returns a new bps Service. The blocklister may be nil. +func New(streamer p2p.Streamer, blocklister p2p.Blocklister, fullNode bool, logger log.Logger, o Options) *Service { return &Service{ fullNode: fullNode, - selfOverlay: overlay, streamer: streamer, + blocklister: blocklister, + opts: o.withDefaults(), + metrics: newMetrics(), cohorts: make(map[string]*cohort), + peers: make(map[string]map[string]int), logger: logger.WithName(loggerName).Register(), } } @@ -82,26 +155,49 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } } -// Interface is the bps surface consumed by the API. It is satisfied by -// *Service and exists so consumers can depend on behaviour, not the concrete -// implementation. +// Close reclaims every cohort and resets their streams. +func (s *Service) Close() error { + s.mtx.Lock() + s.closed = true + cohorts := s.cohorts + s.cohorts = make(map[string]*cohort) + s.mtx.Unlock() + for _, co := range cohorts { + for _, m := range co.shutdown() { + m.reset() + } + } + return nil +} + +// Interface is the bps surface consumed by the API. type Interface interface { - // Join joins a topic's cohort at a broker and returns the live session. + // Join joins a cohort at a broker and returns the live session. Join(ctx context.Context, req JoinRequest) (Session, error) } +// Message is a verified DATA delivery: the chunk and the frame fields its id +// was derived from. +type Message struct { + SOC []byte + Challenge []byte + Index uint64 +} + // Session is a single joined p2p stream to a broker. type Session interface { - // Challenge is the nonce the broker issued for this stream. + // Challenge is the challenge the broker issued for this stream. Every + // chunk published on the session must be signed under it (see ID). Challenge() []byte - // Messages yields raw SOC bytes broadcast by the broker. The channel is - // closed when the stream ends. - Messages() <-chan []byte - // Claim writes the publisher claim SOC to the broker. The broker does not - // reply. - Claim(ctx context.Context, soc []byte) error - // Publish writes a broadcast SOC to the broker. - Publish(ctx context.Context, soc []byte) error + // Messages yields the DATA deliveries that verify against the cohort spec + // and the session cursor. The channel is closed when the stream ends. + Messages() <-chan Message + // Cursor is the lowest DATA index the session will deliver next. + Cursor() uint64 + // Publish writes a chunk of the given kind and index, signed under the + // session challenge, to the broker. The broker does not reply. Only a + // session whose identity is the cohort principal may publish. + Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error // Done is closed when the p2p stream ends. Done() <-chan struct{} // Err reports why the stream ended. Valid after Done is closed. @@ -110,178 +206,156 @@ type Session interface { Close() error } -// TopicBinding is the topic binding carried by Join, wrapped so consumers do -// not depend on the generated protobuf package. -type TopicBinding int - -const ( - BindingUndefined TopicBinding = iota - BindingFeed -) +// CohortSpec names a cohort: the feed of the principal on the topic. The +// topic binding is always FEED_TOPIC. +type CohortSpec struct { + Topic []byte // 32 bytes: the feed topic + Principal []byte // 20 bytes: the publisher +} -// proto maps the wrapped binding to its wire value. -func (b TopicBinding) proto() pb.TopicBinding { - switch b { - case BindingFeed: - return pb.TopicBinding_BINDING_FEED - default: - return pb.TopicBinding_BINDING_UNDEFINED - } +func (c CohortSpec) proto() *pb.CohortSpec { + return &pb.CohortSpec{Topic: c.Topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: c.Principal} } -// JoinRequest describes a cohort join. Identity is the joiner's address, -// injected from the API rather than taken from the peer address, so a node can -// join on behalf of a client. +// JoinRequest describes a cohort join. type JoinRequest struct { - Broker swarm.Address // where to join the topic - Binding TopicBinding // which kind of topic binding - Topic []byte // the topic - Principal []byte // the "owner/admin" - Identity []byte // the joining node's identity + Broker swarm.Address // the broker + Spec CohortSpec // the cohort to join + Identity []byte // 20 bytes: the joining stream's identity + Cursor uint64 // the lowest DATA index to deliver, the subscriber's cursor for the cohort } -// Join onto a topic at the broker. This is called on the subscriber. +// Join joins a cohort at the broker. A peer whose join was refused or whose +// stream was reset must back off before rejoining. func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { + join := &pb.Join{Cohort: req.Spec.proto(), Identity: req.Identity} + if err := validateJoin(join); err != nil { + return nil, err + } stream, err := s.streamer.NewStream(ctx, req.Broker, nil, protocolName, protocolVersion, streamName) if err != nil { return nil, fmt.Errorf("new stream: %w", err) } - ctx, cancel := context.WithCancel(ctx) - w, r := protobuf.NewWriterAndReader(stream) - joinMsg := pb.Join{ - Topic: req.Topic, - Binding: req.Binding.proto(), - Principal: req.Principal, - Identity: req.Identity, + if err := w.WriteMsgWithContext(ctx, join); err != nil { + _ = stream.Reset() + return nil, fmt.Errorf("write join: %w", err) } - if err := w.WriteMsgWithContext(ctx, &joinMsg); err != nil { - cancel() - stream.Reset() - return nil, fmt.Errorf("write join msg: %w", err) + var ack pb.Ack + if err := r.ReadMsgWithContext(ctx, &ack); err != nil { + _ = stream.Reset() + return nil, fmt.Errorf("read ack: %w", err) } - joinAck := pb.JoinAck{} - if err := r.ReadMsgWithContext(ctx, &joinAck); err != nil { - cancel() - stream.Reset() - return nil, fmt.Errorf("read join ack: %w", err) + if ack.Status != pb.Status_OK { + _ = stream.Reset() + return nil, fmt.Errorf("join %s: %w", ack.Status, errJoinRefused) } - if joinAck.Status != pb.Status_STATUS_OK { - cancel() - stream.Reset() - return nil, fmt.Errorf("join %s: %w", joinAck.Status, errJoinRejected) + if len(ack.Challenge) != ChallengeSize { + _ = stream.Reset() + return nil, fmt.Errorf("challenge length %d: %w", len(ack.Challenge), errProtocolError) } + ctx, cancel := context.WithCancel(context.Background()) sess := &session{ ctx: ctx, cancel: cancel, - challenge: joinAck.Challenge, - rx: make(chan []byte, broadcastBufferSize), - tx: make(chan []byte), + w: w, + challenge: ack.Challenge, + publisher: bytes.Equal(req.Identity, req.Spec.Principal), + rx: make(chan Message), done: make(chan struct{}), } + sess.cursor.Store(req.Cursor) - // reading should always yield broadcast msgs - go func() { - defer stream.FullClose() + go sess.readLoop(stream, r, req.Spec) - var readErr error - defer func() { sess.finish(readErr) }() - - for { - msg := pb.Broadcast{} - if err := r.ReadMsgWithContext(ctx, &msg); err != nil { - if ctx.Err() == nil { - s.logger.Error(err, "read broadcast") - } - readErr = err - return - } - - // we might want to do some input validation to see that the broker isn't tricking us + return sess, nil +} - select { - case sess.rx <- msg.Soc: - default: - } +// readLoop delivers the broker's frames until the stream ends. +func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader, spec CohortSpec) { + var readErr error + defer func() { + s.finish(readErr) + if readErr != nil && s.ctx.Err() == nil { + _ = stream.Reset() + } else { + _ = stream.FullClose() } }() - - // the write loop will only be used in the case of a publisher. - // for readers this is essentially a noop. go func() { - defer stream.FullClose() + <-s.ctx.Done() + _ = stream.Reset() + }() - select { - case cl := <-sess.tx: - claim := pb.Broadcast{Soc: cl} - if err := w.WriteMsgWithContext(ctx, &claim); err != nil { - s.logger.Error(err, "write claim") - return - } - case <-ctx.Done(): + for { + var f pb.Broadcast + if err := r.ReadMsgWithContext(s.ctx, &f); err != nil { + readErr = err return } - - for { - select { - case bcast := <-sess.tx: - msg := pb.Broadcast{Soc: bcast} - if err := w.WriteMsgWithContext(ctx, &msg); err != nil { - s.logger.Error(err, "write broadcast") - return - } - case <-ctx.Done(): - return - } + // a retransmit below the cursor is skipped. The challenge is not + // compared: it is the publisher stream's, which the subscriber never sees. + if f.Kind != pb.Kind_DATA || f.Index < s.cursor.Load() { + continue } - }() - - return sess, nil + // re-verify end to end: a broker that forwards a chunk not valid + // for the spec is misbehaving, so the stream is ended + if err := verify(&f, spec.Topic, spec.Principal); err != nil { + readErr = fmt.Errorf("%w: %w", errInvalidDelivery, err) + return + } + s.cursor.Store(f.Index + 1) + select { + case s.rx <- Message{SOC: f.Soc, Challenge: f.Challenge, Index: f.Index}: + case <-s.ctx.Done(): + return + } + } } // session is the concrete Session implementation. type session struct { ctx context.Context cancel context.CancelFunc + wmtx sync.Mutex + w protobuf.Writer challenge []byte - rx chan []byte - tx chan []byte + publisher bool // the identity is the cohort principal + cursor atomic.Uint64 + rx chan Message done chan struct{} err error once sync.Once } -func (s *session) Challenge() []byte { return s.challenge } -func (s *session) Messages() <-chan []byte { return s.rx } -func (s *session) Done() <-chan struct{} { return s.done } -func (s *session) Err() error { return s.err } +func (s *session) Challenge() []byte { return s.challenge } +func (s *session) Messages() <-chan Message { return s.rx } +func (s *session) Cursor() uint64 { return s.cursor.Load() } +func (s *session) Done() <-chan struct{} { return s.done } +func (s *session) Err() error { return s.err } func (s *session) Close() error { s.cancel() return nil } -func (s *session) Claim(ctx context.Context, soc []byte) error { - select { - case s.tx <- soc: - return nil - case <-s.done: - return errSessionClosed - case <-ctx.Done(): - return ctx.Err() +func (s *session) Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error { + if !s.publisher { + return errNotPublisher } -} - -func (s *session) Publish(ctx context.Context, soc []byte) error { select { - case s.tx <- soc: - return nil case <-s.done: return errSessionClosed - case <-ctx.Done(): - return ctx.Err() + default: } + s.wmtx.Lock() + defer s.wmtx.Unlock() + f := pb.Broadcast{Soc: soc, Kind: kind.proto(), Challenge: s.challenge, Index: index} + if err := s.w.WriteMsgWithContext(ctx, &f); err != nil { + return fmt.Errorf("write broadcast: %w", err) + } + return nil } // finish records the terminal error and signals stream completion exactly once. @@ -293,62 +367,75 @@ func (s *session) finish(err error) { }) } +// validateJoin rejects a Join whose spec has a value outside SWIP-74 or whose +// identity is not 20 bytes. +func validateJoin(join *pb.Join) error { + cohortSpec := join.Cohort + switch { + case cohortSpec == nil: + return fmt.Errorf("no cohort: %w", errInvalidJoin) + case len(cohortSpec.Topic) != swarm.HashSize: + return fmt.Errorf("topic length %d: %w", len(cohortSpec.Topic), errInvalidJoin) + case cohortSpec.Binding != pb.TopicBinding_FEED_TOPIC: + return fmt.Errorf("binding %s: %w", cohortSpec.Binding, errInvalidJoin) + case len(cohortSpec.Principal) != crypto.AddressSize: + return fmt.Errorf("principal length %d: %w", len(cohortSpec.Principal), errInvalidJoin) + case len(join.Identity) != crypto.AddressSize: + return fmt.Errorf("identity length %d: %w", len(join.Identity), errInvalidJoin) + } + return nil +} + // handler is the protocol handler on the broker. func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { if !s.fullNode { - stream.Reset() + _ = stream.Reset() return errNotBroker } - ctx, cancel := context.WithCancel(ctx) - defer cancel() - defer stream.FullClose() w, r := protobuf.NewWriterAndReader(stream) var join pb.Join if err := r.ReadMsgWithContext(ctx, &join); err != nil { - return fmt.Errorf("read sys message: %w", err) + _ = stream.Reset() + return fmt.Errorf("read join: %w", err) } - if err := validateJoin(&join); err != nil { - if werr := w.WriteMsgWithContext(ctx, &pb.JoinAck{Status: pb.Status_STATUS_REJECTED}); werr != nil { - return fmt.Errorf("write join ack: %w", werr) - } - stream.Reset() - return err + return s.refuse(ctx, w, stream, pb.Status_REJECTED, err) } + // re-marshal so that the cohort key ignores fields BPS-lite does not define + join.Cohort = &pb.CohortSpec{Topic: join.Cohort.Topic, Binding: join.Cohort.Binding, Principal: join.Cohort.Principal} + + ctx, cancel := context.WithCancel(ctx) + defer cancel() - // add to the cohort and get a channel to receive the broadcasts on - ch, challenge, err := s.joinCohort(&join) + co, m, err := s.join(p.Address, &join, func() { + cancel() + _ = stream.Reset() + }) if err != nil { - if werr := w.WriteMsgWithContext(ctx, &pb.JoinAck{Status: pb.Status_STATUS_REJECTED}); werr != nil { - return fmt.Errorf("write join ack: %w", werr) + status := pb.Status_REJECTED + if errors.Is(err, errFull) { + status = pb.Status_FULL } - stream.Reset() - return err - } - // register in the cohort and return the secret - // peer is trying to join the cohort. accept and return the challenge - ack := pb.JoinAck{Challenge: challenge, Status: pb.Status_STATUS_OK} - if err := w.WriteMsgWithContext(ctx, &ack); err != nil { - s.left(join.Identity, join.Topic) - return fmt.Errorf("write claim: %w", err) - } - - defer s.left(join.Identity, join.Topic) - var wg sync.WaitGroup - // the writer side - reads messages off the publisher channel - // and pushes them to the subscriber stream - wg.Add(1) + return s.refuse(ctx, w, stream, status, err) + } + // detach on every exit, normal or not + defer s.leave(p.Address, co, m) + + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: m.challenge}); err != nil { + m.reset() + return fmt.Errorf("write ack: %w", err) + } + + // the writer side: deliveries to a subscriber stream + writerDone := make(chan struct{}) go func() { - defer wg.Done() - defer cancel() + defer close(writerDone) for { - // await messages, then write to the stream once they come in select { - case msg := <-ch: - m := pb.Broadcast{Soc: msg} - if err := w.WriteMsgWithContext(ctx, &m); err != nil { - s.logger.Error(err, "write broadcast") + case f := <-m.queue: + if err := w.WriteMsgWithContext(ctx, f); err != nil { + m.reset() return } case <-ctx.Done(): @@ -356,199 +443,324 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } } }() - - // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off - // check if claim signature pk matches the feed owner address on the registry - // if it does - this is an atomic swap - the current stream becomes the publisher stream - // and the next challenge changes randomly, so that if needed, the publisher can reclaim - // later and rejoin + reclaim. - - // writer side - we first try to read a claim. we can wait indefinitely here. - // once a claim is accepted, we prompte the stream to be a publisher stream - // then continuously try to read from it broadcast messages and push them over the - // publisher channel - wg.Add(1) - go func() { - defer wg.Done() - defer cancel() - claim := pb.Broadcast{} - if err := r.ReadMsgWithContext(ctx, &claim); err != nil { - s.logger.Error(err, "read claim") - return - } - - // do the checks on the claim, if it is wrong then we reset the stream - ch1, err := s.claim(&join, claim.Soc) - if err != nil { - stream.Reset() - return - } - defer close(ch1) - for { - // await messages, then write to the cohort channel - m := pb.Broadcast{} - if err := r.ReadMsgWithContext(ctx, &m); err != nil { - s.logger.Error(err, "read broadcast") - return + defer func() { <-writerDone }() + + // the reader side: publications and authentications + for { + var f pb.Broadcast + if err := r.ReadMsgWithContext(ctx, &f); err != nil { + if ctx.Err() != nil { + return nil } - select { - case ch1 <- m.Soc: - case <-ctx.Done(): - return + m.reset() + return nil + } + if err := s.handleFrame(co, m, &f); err != nil { + if errors.Is(err, errViolation) && s.blocklister != nil { + if berr := s.blocklister.Blocklist(p.Address, s.opts.ViolationBlocklist, err.Error()); berr != nil { + s.logger.Debug("blocklist failed", "peer_address", p.Address, "error", berr) + } } + m.reset() + return err } - }() - wg.Wait() - return nil + } } -var ( - errInvalidProof = errors.New("bps: invalid proof") - errInvalidTopic = errors.New("bps: invalid topic") - errInvalidPrincipal = errors.New("bps: invalid principal") - errInvalidIdentity = errors.New("bps: invalid identity") - errCohortMismatch = errors.New("bps: cohort binding mismatch") -) +// refuse answers a Join with a non-OK Ack, which ends the stream. +func (s *Service) refuse(ctx context.Context, w protobuf.Writer, stream p2p.Stream, status pb.Status, cause error) error { + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: status}); err != nil { + _ = stream.Reset() + return fmt.Errorf("write ack: %w", err) + } + _ = stream.FullClose() + s.logger.Debug("join refused", "status", status, "error", cause) + return nil +} -// validateJoin rejects a Join naming malformed or unsupported fields. -func validateJoin(join *pb.Join) error { - if len(join.Topic) != swarm.HashSize { - return errInvalidTopic +// handleFrame validates a Broadcast arriving at the broker per SWIP-74 and +// accepts, drops or punishes it. A returned error resets the stream. +func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { + co.mtx.Lock() + role, cursor := m.role, co.cursor + co.mtx.Unlock() + + // 1. only a publisher or pending stream may send + if role == roleSubscriber { + s.count(&co.counters.wrongStream, "wrong_stream") + return fmt.Errorf("%w: %w", errViolation, errWrongStream) + } + // 2. a kind BPS-lite does not define is dropped + if f.Kind != pb.Kind_DATA && f.Kind != pb.Kind_AUTH { + s.count(&co.counters.unknownKind, "unknown_kind") + return nil } - if len(join.Principal) != crypto.AddressSize { - return errInvalidPrincipal + // 3. the challenge must be the stream's + if !bytes.Equal(f.Challenge, m.challenge) { + s.count(&co.counters.wrongChallenge, "wrong_challenge") + return nil } - if len(join.Identity) != crypto.AddressSize { - return errInvalidIdentity + // 4. on a publisher stream a DATA frame below the cursor is a retransmit; + // on a pending stream it is checked after the signature, so that it authenticates + if role == rolePublisher && f.Kind == pb.Kind_DATA && f.Index < cursor { + s.count(&co.counters.retransmit, "retransmit") + return nil } - if _, err := bindingFor(join.Binding); err != nil { - return err + // 5. the chunk must be the principal's at the id derived from the frame + if err := verify(f, co.spec.Topic, co.spec.Principal); err != nil { + s.count(&co.counters.invalidSOC, "invalid_soc") + return fmt.Errorf("%w: %w", errViolation, err) + } + + var overflow []*member + co.mtx.Lock() + if m.role == rolePending { + m.role = rolePublisher + m.authTimer.Stop() + } + switch { + case f.Kind == pb.Kind_AUTH: + co.lastActivity = time.Now() + case f.Index < co.cursor: + s.count(&co.counters.retransmit, "retransmit") + default: + co.cursor = f.Index + 1 + co.lastActivity = time.Now() + s.metrics.Delivered.Inc() + for sub := range co.members { + if sub.role != roleSubscriber { + continue + } + select { + case sub.queue <- f: + default: + s.count(&co.counters.queueReset, "queue_reset") + overflow = append(overflow, sub) + } + } + } + co.mtx.Unlock() + for _, sub := range overflow { + sub.reset() } return nil } -// join/open operation in one - the peer either joins an existing -// cohort or creates one by joining a previously unregistered topic. -// returns the cohort challenge and the channel that data will be sent over -func (s *Service) joinCohort(join *pb.Join) (chan []byte, []byte, error) { +// join attaches a stream to the cohort named by the spec, creating the +// cohort if no live one has it. An error wrapping errFull means a capacity +// bound was hit. +func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort, *member, error) { + key, err := join.Cohort.Marshal() + if err != nil { + return nil, nil, fmt.Errorf("marshal cohort spec: %w", err) + } + k := string(key) + s.mtx.Lock() defer s.mtx.Unlock() - t := string(join.Topic) - co, ok := s.cohorts[t] + if s.closed { + return nil, nil, errSessionClosed + } - m := newMember(join.Identity) - if ok { - if co.binding != join.Binding || !bytes.Equal(co.principal, join.Principal) { - return nil, nil, errCohortMismatch - } - co.members[string(join.Identity)] = m - return m.ch, m.challenge, nil + pc := s.peers[peer.ByteString()] + if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerPeer { + return nil, nil, fmt.Errorf("too many cohorts per peer: %w", errFull) } - members := make(map[string]*member) - members[string(join.Identity)] = m - s.cohorts[t] = &cohort{ - topic: join.Topic, - binding: join.Binding, - principal: join.Principal, - members: members, + if pc[k] >= s.opts.MaxStreamsPerPeerCohort { + return nil, nil, fmt.Errorf("too many streams per peer per cohort: %w", errFull) } - return m.ch, m.challenge, nil -} + role := roleSubscriber + if bytes.Equal(join.Identity, join.Cohort.Principal) { + role = rolePending + } -func (s *Service) left(identity, topic []byte) { - s.mtx.Lock() - defer s.mtx.Unlock() - t := string(topic) - co, ok := s.cohorts[t] - if ok { - delete(co.members, string(identity)) + co, ok := s.cohorts[k] + if !ok { + if len(s.cohorts) >= s.opts.MaxCohorts { + return nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) + } + co = &cohort{ + key: k, + spec: join.Cohort, + members: make(map[*member]struct{}), + lastActivity: time.Now(), + } + co.inactivity = time.AfterFunc(s.opts.InactivityTimeout, func() { s.reclaim(co) }) + s.cohorts[k] = co + s.metrics.Cohorts.Set(float64(len(s.cohorts))) + } + + co.mtx.Lock() + defer co.mtx.Unlock() + if role == roleSubscriber && co.subscribers >= s.opts.MaxSubscribers { if len(co.members) == 0 { - delete(s.cohorts, t) + s.removeCohort(co) } + return nil, nil, fmt.Errorf("too many subscribers per cohort: %w", errFull) } + + challenge := make([]byte, ChallengeSize) + if _, err := rand.Read(challenge); err != nil { + if len(co.members) == 0 { + s.removeCohort(co) + } + return nil, nil, fmt.Errorf("challenge: %w", err) + } + m := &member{ + role: role, + challenge: challenge, + queue: make(chan *pb.Broadcast, s.opts.QueueSize), + } + var once sync.Once + m.reset = func() { once.Do(reset) } + if role == rolePending { + m.authTimer = time.AfterFunc(s.opts.AuthTimeout, func() { s.authTimeout(peer, co, m) }) + } else { + co.subscribers++ + } + co.members[m] = struct{}{} + + if pc == nil { + pc = make(map[string]int) + s.peers[peer.ByteString()] = pc + } + pc[k]++ + return co, m, nil } -func (s *Service) claim(join *pb.Join, socBlob []byte) (chan []byte, error) { +// leave detaches a stream. A cohort with no attached streams is reclaimed at once. +func (s *Service) leave(peer swarm.Address, co *cohort, m *member) { s.mtx.Lock() defer s.mtx.Unlock() - t := string(join.Topic) - co, ok := s.cohorts[t] - if !ok { - return nil, errors.New("tried to claim non-existent topic") + + if pc := s.peers[peer.ByteString()]; pc != nil { + if pc[co.key]--; pc[co.key] <= 0 { + delete(pc, co.key) + } + if len(pc) == 0 { + delete(s.peers, peer.ByteString()) + } } - member, ok := co.members[string(join.Identity)] - if !ok { - return nil, errors.New("no such member") + + co.mtx.Lock() + defer co.mtx.Unlock() + if _, ok := co.members[m]; !ok { + return } - b, err := bindingFor(co.binding) - if err != nil { - return nil, err + delete(co.members, m) + if m.authTimer != nil { + m.authTimer.Stop() } - addr, err := b.claimAddress(co.topic, co.principal) - if err != nil { - return nil, fmt.Errorf("claim address: %w", err) + if m.role == roleSubscriber { + co.subscribers-- } - chunk := swarm.NewChunk(addr, socBlob) - if !soc.Valid(chunk) { - return nil, errInvalidProof + if len(co.members) == 0 && s.cohorts[co.key] == co { + s.removeCohort(co) } - proofSoc, err := soc.FromChunk(chunk) - if err != nil { - return nil, fmt.Errorf("claim soc: %w", err) +} + +// removeCohort forgets the cohort. Must be called with s.mtx and co.mtx held. +func (s *Service) removeCohort(co *cohort) { + delete(s.cohorts, co.key) + co.inactivity.Stop() + s.metrics.Cohorts.Set(float64(len(s.cohorts))) +} + +// reclaim forgets a cohort on which no frame was accepted for the inactivity +// deadline, and resets every stream in it. +func (s *Service) reclaim(co *cohort) { + s.mtx.Lock() + if s.cohorts[co.key] != co { + s.mtx.Unlock() + return } - if err := b.verifyClaim(proofSoc, co.principal, member.challenge, s.selfOverlay.Bytes(), co.topic); err != nil { - return nil, err + co.mtx.Lock() + if idle := time.Since(co.lastActivity); idle < s.opts.InactivityTimeout { + co.inactivity.Reset(s.opts.InactivityTimeout - idle) + co.mtx.Unlock() + s.mtx.Unlock() + return } + s.removeCohort(co) + co.mtx.Unlock() + s.mtx.Unlock() - publisher := member - txCh := make(chan []byte) - go func() { - for v := range txCh { - s.mtx.Lock() - co, ok := s.cohorts[t] - if !ok { - s.mtx.Unlock() - return - } - for _, m := range co.members { - if m == publisher { - continue - } - select { - case m.ch <- v: - default: - } - } - s.mtx.Unlock() - } - }() - return txCh, nil + s.logger.Debug("cohort reclaimed", "error", errCohortReclaimed) + for _, m := range co.shutdown() { + m.reset() + } } -type cohort struct { - topic []byte // topic is the feed topic - binding pb.TopicBinding - principal []byte // governing identity allowed to publish - // lastSeen uint64 // last feed update index, used to prevent replay and circumvent dedup logic (for now) - members map[string]*member +// authTimeout disconnects a pending stream that has not authenticated in time. +func (s *Service) authTimeout(peer swarm.Address, co *cohort, m *member) { + co.mtx.Lock() + pending := m.role == rolePending + co.mtx.Unlock() + if !pending { + return + } + s.count(&co.counters.authTimeout, "auth_timeout") + if s.blocklister != nil { + if err := s.blocklister.Blocklist(peer, s.opts.BlocklistDuration, "bps auth timeout"); err != nil { + s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) + } + } + m.reset() } -func newMember(identity []byte) *member { - challenge := make([]byte, 32) - if _, err := rand.Read(challenge); err != nil { - panic(err) +// Counters returns the counters of the live cohort named by the spec. +func (s *Service) Counters(spec CohortSpec) (Counters, bool) { + key, err := spec.proto().Marshal() + if err != nil { + return Counters{}, false } + s.mtx.Lock() + co, ok := s.cohorts[string(key)] + s.mtx.Unlock() + if !ok { + return Counters{}, false + } + return co.counters.snapshot(), true +} - return &member{ - identity: identity, - ch: make(chan []byte, broadcastBufferSize), - challenge: challenge, +type role int + +const ( + roleSubscriber role = iota + 1 // receives, never publishes + rolePending // declared the principal as identity, not authenticated yet + rolePublisher // authenticated: may publish, receives nothing +) + +type cohort struct { + key string + spec *pb.CohortSpec + inactivity *time.Timer + counters counters + + mtx sync.Mutex + cursor uint64 // the lowest DATA index accepted next + lastActivity time.Time + members map[*member]struct{} + subscribers int +} + +// shutdown returns the cohort's streams, for the caller to reset. +func (co *cohort) shutdown() []*member { + co.mtx.Lock() + defer co.mtx.Unlock() + ms := make([]*member, 0, len(co.members)) + for m := range co.members { + ms = append(ms, m) } + return ms } +// member is one stream attached to a cohort. role is guarded by cohort.mtx. type member struct { - identity []byte - ch chan []byte + role role challenge []byte + queue chan *pb.Broadcast + reset func() + authTimer *time.Timer } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index fb9b67c6fe6..4fb14fc7686 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -7,98 +7,546 @@ package bps_test import ( "bytes" "context" + "errors" + "sync" "testing" "time" "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/bps/pb" "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" "github.com/ethersphere/bee/v2/pkg/p2p/streamtest" "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" ) -func TestClaimAndBroadcast(t *testing.T) { - t.Parallel() +const waitFor = 5 * time.Second - logger := log.Noop +type blocklister struct { + mu sync.Mutex + peers map[string]string +} - brokerAddr := swarm.RandAddress(t) - broker := bps.New(nil, brokerAddr, true, logger) +func (b *blocklister) NetworkStatus() p2p.NetworkStatus { return p2p.NetworkStatusAvailable } - // each client gets its own recorder so that the broker sees distinct peer overlays - newClient := func() *bps.Service { - recorder := streamtest.New( - streamtest.WithProtocols(broker.Protocol()), - streamtest.WithBaseAddr(swarm.RandAddress(t)), - ) - return bps.New(recorder, swarm.RandAddress(t), false, logger) - } - publisher := newClient() - subscriber := newClient() +func (b *blocklister) Blocklist(overlay swarm.Address, _ time.Duration, reason string) error { + b.mu.Lock() + defer b.mu.Unlock() + b.peers[overlay.ByteString()] = reason + return nil +} - // the public topic is the soc address of the claim: keccak(id | owner) +func (b *blocklister) blocklisted(overlay swarm.Address) bool { + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.peers[overlay.ByteString()] + return ok +} + +type env struct { + t *testing.T + broker *bps.Service + brokerAddr swarm.Address + bl *blocklister + signer crypto.Signer + principal []byte + topic []byte +} + +func newEnv(t *testing.T, o bps.Options) *env { + t.Helper() + bl := &blocklister{peers: make(map[string]string)} + broker := bps.New(nil, bl, true, log.Noop, o) + t.Cleanup(func() { _ = broker.Close() }) key, err := crypto.GenerateSecp256k1Key() if err != nil { t.Fatal(err) } signer := crypto.NewDefaultSigner(key) - owner, err := crypto.NewEthereumAddress(key.PublicKey) - if err != nil { - t.Fatal(err) - } - id := make([]byte, swarm.HashSize) - copy(id, "bps-test-topic") - topic, err := soc.CreateAddress(id, owner) + admin, err := signer.EthereumAddress() if err != nil { t.Fatal(err) } + topic := make([]byte, swarm.HashSize) + copy(topic, "bps-test-topic") + return &env{t: t, broker: broker, brokerAddr: swarm.RandAddress(t), bl: bl, signer: signer, principal: admin.Bytes(), topic: topic} +} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +// client returns a node whose streams reach the broker from overlay. +func (e *env) client(overlay swarm.Address) (*bps.Service, *streamtest.Recorder) { + rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(overlay)) + return bps.New(rec, nil, false, log.Noop, bps.Options{}), rec +} - _, subRx, _, _, err := subscriber.Join(ctx, brokerAddr, topic.Bytes()) +func (e *env) join(c *bps.Service, addr []byte) bps.Session { + e.t.Helper() + s, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: addr}) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } + e.t.Cleanup(func() { _ = s.Close() }) + return s +} + +func (e *env) subscribe() bps.Session { + e.t.Helper() + c, _ := e.client(swarm.RandAddress(e.t)) + addr := make([]byte, 20) + copy(addr, swarm.RandAddress(e.t).Bytes()) + return e.join(c, addr) +} - challenge, pubRx, pubTx, claim, err := publisher.Join(ctx, brokerAddr, topic.Bytes()) +func (e *env) publisher() (bps.Session, swarm.Address) { + e.t.Helper() + overlay := swarm.RandAddress(e.t) + c, _ := e.client(overlay) + return e.join(c, e.principal), overlay +} + +func (e *env) chunk(signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { + e.t.Helper() + id, err := bps.ID(kind, e.topic, challenge, index) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } - - // claim payload: challenge | broker overlay - payload := append(append([]byte{}, challenge...), brokerAddr.Bytes()...) ch, err := cac.New(payload) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } - proof, err := soc.New(id, ch).Sign(signer) + s, err := soc.New(id, ch).Sign(signer) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } - claim(proof.Data()) + return s.Data() +} - msg := []byte("hello cohort") - select { - case pubTx <- msg: - case <-time.After(time.Second): - t.Fatal("timed out sending broadcast") +func (e *env) publish(s bps.Session, kind bps.Kind, index uint64, payload []byte) []byte { + e.t.Helper() + c := e.chunk(e.signer, kind, s.Challenge(), index, payload) + if err := s.Publish(context.Background(), kind, index, c); err != nil { + e.t.Fatal(err) } + return c +} + +func (e *env) counters() bps.Counters { + e.t.Helper() + c, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}) + if !ok { + e.t.Fatal("no live cohort") + } + return c +} +func expectMessage(t *testing.T, s bps.Session, index uint64, chunk []byte) { + t.Helper() select { - case got := <-subRx: - if !bytes.Equal(got, msg) { - t.Fatalf("got message %q, want %q", got, msg) + case m := <-s.Messages(): + if m.Index != index || !bytes.Equal(m.SOC, chunk) { + t.Fatalf("got index %d %x, want %d %x", m.Index, m.SOC, index, chunk) } - case <-time.After(time.Second): - t.Fatal("timed out waiting for subscriber to receive broadcast") + case <-time.After(waitFor): + t.Fatalf("timed out waiting for index %d", index) } +} +func expectNoMessage(t *testing.T, s bps.Session) { + t.Helper() select { - case got := <-pubRx: - t.Fatalf("publisher received its own broadcast %q", got) + case m, ok := <-s.Messages(): + if ok { + t.Fatalf("unexpected message at index %d", m.Index) + } case <-time.After(100 * time.Millisecond): } } + +func expectDone(t *testing.T, s bps.Session) { + t.Helper() + select { + case <-s.Done(): + case <-time.After(waitFor): + t.Fatal("stream not ended") + } +} + +func eventually(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(waitFor) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not met") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestBroadcast(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, _ := e.publisher() + + if len(pub.Challenge()) != bps.ChallengeSize || bytes.Equal(pub.Challenge(), sub.Challenge()) { + t.Fatal("want a distinct 32-byte challenge per stream") + } + + // AUTH authenticates and is never delivered + e.publish(pub, bps.KindAuth, 0, nil) + c0 := e.publish(pub, bps.KindData, 0, []byte("hello cohort")) + expectMessage(t, sub, 0, c0) + expectNoMessage(t, pub) +} + +func TestCursor(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, _ := e.publisher() + + c5 := e.publish(pub, bps.KindData, 5, []byte("five")) // gaps are allowed + expectMessage(t, sub, 5, c5) + e.publish(pub, bps.KindData, 3, []byte("three")) // retransmit + e.publish(pub, bps.KindData, 5, []byte("five")) // retransmit + c6 := e.publish(pub, bps.KindData, 6, []byte("six")) + expectMessage(t, sub, 6, c6) + if got := e.counters().Retransmit; got != 2 { + t.Fatalf("retransmit %d, want 2", got) + } +} + +func TestPublisherReconnects(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + + pub1, _ := e.publisher() + for i := uint64(0); i < 3; i++ { + expectMessage(t, sub, i, e.publish(pub1, bps.KindData, i, []byte{byte(i)})) + } + _ = pub1.Close() + expectDone(t, pub1) + + // the cohort and its cursor persist; a valid frame below the cursor + // still authenticates the new stream and is counted as a retransmit + pub2, _ := e.publisher() + if bytes.Equal(pub1.Challenge(), pub2.Challenge()) { + t.Fatal("challenge reused") + } + e.publish(pub2, bps.KindData, 1, []byte{1}) + c3 := e.publish(pub2, bps.KindData, 3, []byte{3}) + expectMessage(t, sub, 3, c3) + if got := e.counters().Retransmit; got != 1 { + t.Fatalf("retransmit %d, want 1", got) + } +} + +func TestTwoPublisherStreams(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub1, _ := e.publisher() + pub2, _ := e.publisher() + + c0 := e.publish(pub1, bps.KindData, 0, []byte("a")) + expectMessage(t, sub, 0, c0) + e.publish(pub2, bps.KindData, 0, []byte("a")) // signed for its own stream, delivered once + c1 := e.publish(pub2, bps.KindData, 1, []byte("b")) + expectMessage(t, sub, 1, c1) + expectNoMessage(t, pub1) +} + +func TestViolations(t *testing.T) { + t.Parallel() + + other, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + otherSigner := crypto.NewDefaultSigner(other) + + t.Run("publish on a subscriber session", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + c := e.chunk(e.signer, bps.KindData, sub.Challenge(), 0, []byte("x")) + if err := sub.Publish(context.Background(), bps.KindData, 0, c); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("got %v, want %v", err, bps.ErrNotPublisher) + } + }) + + t.Run("publication from a subscriber stream", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + overlay := swarm.RandAddress(t) + keep := e.subscribe() + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Identity: swarm.RandAddress(t).Bytes()[:20]}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + c := e.chunk(e.signer, bps.KindData, ack.Challenge, 0, []byte("x")) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ack.Challenge, Soc: c}); err != nil { + t.Fatal(err) + } + if err := r.ReadMsg(&pb.Broadcast{}); err == nil { + t.Fatal("stream not ended") + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().WrongStream; got != 1 { + t.Fatalf("wrong_stream %d, want 1", got) + } + expectNoMessage(t, keep) + }) + + for _, tc := range []struct { + name string + chunk func(e *env, challenge []byte) (bps.Kind, uint64, []byte) + }{ + {"wrong signer", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(otherSigner, bps.KindData, ch, 0, []byte("x")) + }}, + {"signed for another challenge", func(e *env, _ []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(e.signer, bps.KindData, make([]byte, 32), 0, []byte("x")) + }}, + {"index not the signed one", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 1, e.chunk(e.signer, bps.KindData, ch, 0, []byte("x")) + }}, + {"auth relabelled as data", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(e.signer, bps.KindAuth, ch, 0, nil) + }}, + {"auth with payload", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindAuth, 0, e.chunk(e.signer, bps.KindAuth, ch, 0, []byte("x")) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, overlay := e.publisher() + kind, index, chunk := tc.chunk(e, pub.Challenge()) + if err := pub.Publish(context.Background(), kind, index, chunk); err != nil { + t.Fatal(err) + } + expectDone(t, pub) + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().InvalidSOC; got != 1 { + t.Fatalf("invalid_soc %d, want 1", got) + } + expectNoMessage(t, sub) + }) + } +} + +// raw opens a stream to the broker and joins with join, returning the Ack. +func (e *env) raw(join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { + e.t.Helper() + return e.rawFrom(swarm.RandAddress(e.t), join) +} + +// rawFrom is raw from the given overlay. +func (e *env) rawFrom(overlay swarm.Address, join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { + e.t.Helper() + rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(overlay)) + stream, err := rec.NewStream(context.Background(), e.brokerAddr, nil, bps.ProtocolName, bps.ProtocolVersion, bps.StreamName) + if err != nil { + e.t.Fatal(err) + } + e.t.Cleanup(func() { _ = stream.Reset() }) + w, r := protobuf.NewWriterAndReader(stream) + if err := w.WriteMsg(join); err != nil { + e.t.Fatal(err) + } + var ack pb.Ack + if err := r.ReadMsg(&ack); err != nil { + e.t.Fatal(err) + } + return w, r, &ack +} + +func (e *env) spec() *pb.CohortSpec { + return &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: e.principal} +} + +func TestDroppedFrames(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + w, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Identity: e.principal}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + write := func(f *pb.Broadcast) { + t.Helper() + if err := w.WriteMsg(f); err != nil { + t.Fatal(err) + } + } + + // another kind and another challenge are dropped, not punished + write(&pb.Broadcast{Kind: pb.Kind(7), Challenge: ack.Challenge, Soc: []byte("roster")}) + other := make([]byte, 32) + write(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: other, Soc: e.chunk(e.signer, bps.KindData, other, 0, []byte("old"))}) + c := e.chunk(e.signer, bps.KindData, ack.Challenge, 0, []byte("ok")) + write(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ack.Challenge, Index: 0, Soc: c}) + expectMessage(t, sub, 0, c) + + got := e.counters() + if got.UnknownKind != 1 || got.WrongChallenge != 1 || got.InvalidSOC != 0 { + t.Fatalf("counters %+v", got) + } +} + +// TestInvalidDelivery checks that a subscriber ends the session when the +// broker forwards a chunk that does not verify against the cohort spec. +func TestInvalidDelivery(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + other, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + challenge := make([]byte, bps.ChallengeSize) + bad := e.chunk(crypto.NewDefaultSigner(other), bps.KindData, challenge, 0, []byte("x")) + broker := p2p.ProtocolSpec{ + Name: bps.ProtocolName, + Version: bps.ProtocolVersion, + StreamSpecs: []p2p.StreamSpec{{ + Name: bps.StreamName, + Handler: func(ctx context.Context, _ p2p.Peer, stream p2p.Stream) error { + w, r := protobuf.NewWriterAndReader(stream) + if err := r.ReadMsgWithContext(ctx, &pb.Join{}); err != nil { + return err + } + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: challenge}); err != nil { + return err + } + if err := w.WriteMsgWithContext(ctx, &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: challenge, Soc: bad}); err != nil { + return err + } + // hold the stream until the subscriber ends it + _ = r.ReadMsgWithContext(ctx, &pb.Broadcast{}) + return nil + }, + }}, + } + rec := streamtest.New(streamtest.WithProtocols(broker), streamtest.WithBaseAddr(swarm.RandAddress(t))) + c := bps.New(rec, nil, false, log.Noop, bps.Options{}) + sub := e.join(c, swarm.RandAddress(t).Bytes()[:20]) + expectDone(t, sub) + if err := sub.Err(); !errors.Is(err, bps.ErrInvalidDelivery) { + t.Fatalf("got %v, want %v", err, bps.ErrInvalidDelivery) + } +} + +func TestJoinRejected(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + for _, tc := range []struct { + name string + join *pb.Join + }{ + {"no cohort", &pb.Join{Identity: e.principal}}, + {"unspecified binding", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: e.principal}}, + {"short topic", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic[:31], Binding: pb.TopicBinding_FEED_TOPIC, Principal: e.principal}, Identity: e.principal}}, + {"no admin", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC}, Identity: e.principal}}, + {"short addr", &pb.Join{Cohort: e.spec(), Identity: e.principal[:19]}}, + } { + _, _, ack := e.raw(tc.join) + if ack.Status != pb.Status_REJECTED || len(ack.Challenge) != 0 { + t.Fatalf("%s: got %s", tc.name, ack.Status) + } + } +} + +func TestBounds(t *testing.T) { + t.Parallel() + + t.Run("subscribers per cohort, principal admitted outside", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxSubscribers: 1}) + sub := e.subscribe() + if _, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Identity: make([]byte, 20)}); ack.Status != pb.Status_FULL { + t.Fatalf("got %s, want FULL", ack.Status) + } + pub, _ := e.publisher() + expectMessage(t, sub, 0, e.publish(pub, bps.KindData, 0, []byte("x"))) + }) + + t.Run("cohorts per broker", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxCohorts: 1}) + e.subscribe() + spec := e.spec() + spec.Topic = make([]byte, 32) + if _, _, ack := e.raw(&pb.Join{Cohort: spec, Identity: e.principal}); ack.Status != pb.Status_FULL { + t.Fatalf("got %s, want FULL", ack.Status) + } + }) + + t.Run("streams and cohorts per peer", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxStreamsPerPeerCohort: 1, MaxCohortsPerPeer: 1}) + c, _ := e.client(swarm.RandAddress(t)) + e.join(c, e.principal) + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: e.principal}); err == nil { + t.Fatal("want a second stream on the cohort refused") + } + other := make([]byte, 32) + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: other, Principal: e.principal}, Identity: e.principal}); err == nil { + t.Fatal("want a second cohort refused") + } + }) + + t.Run("auth timeout", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{AuthTimeout: 50 * time.Millisecond}) + e.subscribe() + pub, overlay := e.publisher() + expectDone(t, pub) + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().AuthTimeout; got != 1 { + t.Fatalf("auth_timeout %d, want 1", got) + } + }) + + t.Run("inactivity deadline", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{InactivityTimeout: 100 * time.Millisecond}) + sub := e.subscribe() + pub, _ := e.publisher() + e.publish(pub, bps.KindAuth, 0, nil) + expectDone(t, sub) + expectDone(t, pub) + if _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}); ok { + t.Fatal("cohort not reclaimed") + } + }) +} + +func TestVerify(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + challenge := make([]byte, 32) + copy(challenge, "challenge") + + c := e.chunk(e.signer, bps.KindData, challenge, 9, []byte("x")) + if err := bps.Verify(bps.KindData, challenge, 9, c, e.topic, e.principal); err != nil { + t.Fatal(err) + } + other := make([]byte, 20) + for _, err := range []error{ + bps.Verify(bps.KindData, challenge, 9, c, e.topic, other), + bps.Verify(bps.KindData, challenge[:31], 9, c, e.topic, e.principal), + bps.Verify(bps.KindAuth, challenge, 9, c, e.topic, e.principal), + bps.Verify(bps.KindData, challenge, 9, c, make([]byte, 32), e.principal), + bps.Verify(bps.KindData, challenge, 9, c[:10], e.topic, e.principal), + } { + if !errors.Is(err, bps.ErrInvalidSOC) { + t.Fatalf("got %v, want invalid soc", err) + } + } +} diff --git a/pkg/bps/export_test.go b/pkg/bps/export_test.go new file mode 100644 index 00000000000..58f68d2fa7e --- /dev/null +++ b/pkg/bps/export_test.go @@ -0,0 +1,17 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +const ( + ProtocolName = protocolName + ProtocolVersion = protocolVersion + StreamName = streamName +) + +var ( + ErrInvalidSOC = errInvalidSOC + ErrNotPublisher = errNotPublisher + ErrInvalidDelivery = errInvalidDelivery +) diff --git a/pkg/bps/id.go b/pkg/bps/id.go new file mode 100644 index 00000000000..001a373779e --- /dev/null +++ b/pkg/bps/id.go @@ -0,0 +1,111 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// ChallengeSize is the length of the per-stream challenge issued in Ack. +const ChallengeSize = 32 + +// servicePrefix domain-separates the session feeds of every kind but DATA. +var servicePrefix = []byte("bps-service:v1") + +// errInvalidSOC is returned for a chunk that does not have the id derived +// from its frame, or does not validate as the principal's single-owner chunk. +var errInvalidSOC = errors.New("bps: invalid soc") + +// Kind is what a chunk on the wire is. +type Kind int + +const ( + KindUnspecified Kind = iota + KindData // an update of the stream's session feed + KindAuth // an empty chunk that authenticates the stream +) + +func (k Kind) proto() pb.Kind { + switch k { + case KindData: + return pb.Kind_DATA + case KindAuth: + return pb.Kind_AUTH + default: + return pb.Kind_KIND_UNSPECIFIED + } +} + +// SessionTopic returns the topic of the session feed of the given kind: +// keccak256(prefix | topic | challenge), with the prefix empty for DATA and +// "bps-service:v1" | kind for any other kind. +func SessionTopic(kind Kind, topic, challenge []byte) ([]byte, error) { + return sessionTopic(kind.proto(), topic, challenge) +} + +// ID returns the single-owner chunk id of the session feed update at index: +// keccak256(SessionTopic(kind, topic, challenge) | index), index big-endian. +func ID(kind Kind, topic, challenge []byte, index uint64) ([]byte, error) { + return id(kind.proto(), topic, challenge, index) +} + +func sessionTopic(kind pb.Kind, topic, challenge []byte) ([]byte, error) { + var prefix []byte + if kind != pb.Kind_DATA { + prefix = append(append([]byte{}, servicePrefix...), byte(kind)) + } + return crypto.LegacyKeccak256(append(append(prefix, topic...), challenge...)) +} + +func id(kind pb.Kind, topic, challenge []byte, index uint64) ([]byte, error) { + topicS, err := sessionTopic(kind, topic, challenge) + if err != nil { + return nil, err + } + return crypto.LegacyKeccak256(binary.BigEndian.AppendUint64(topicS, index)) +} + +// Verify checks that the frame fields kind, challenge and index give the id of +// the chunk, and that the chunk validates as the principal's single-owner chunk +// at keccak256(id | principal). An AUTH chunk must moreover be empty. +func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, principal []byte) error { + return verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, principal) +} + +func verify(f *pb.Broadcast, topic, principal []byte) error { + if len(f.Challenge) != ChallengeSize { + return fmt.Errorf("challenge length %d: %w", len(f.Challenge), errInvalidSOC) + } + want, err := id(f.Kind, topic, f.Challenge, f.Index) + if err != nil { + return err + } + if len(f.Soc) < swarm.HashSize || !bytes.Equal(f.Soc[:swarm.HashSize], want) { + return fmt.Errorf("id not derived from frame: %w", errInvalidSOC) + } + addr, err := soc.CreateAddress(want, principal) + if err != nil { + return err + } + if !soc.Valid(swarm.NewChunk(addr, f.Soc)) { + return fmt.Errorf("not the principal's soc: %w", errInvalidSOC) + } + if f.Kind == pb.Kind_AUTH { + // span 0, no payload + if len(f.Soc) != swarm.HashSize+swarm.SocSignatureSize+swarm.SpanSize || + binary.LittleEndian.Uint64(f.Soc[swarm.HashSize+swarm.SocSignatureSize:]) != 0 { + return fmt.Errorf("auth chunk not empty: %w", errInvalidSOC) + } + } + return nil +} diff --git a/pkg/bps/metrics.go b/pkg/bps/metrics.go new file mode 100644 index 00000000000..0473653e6dd --- /dev/null +++ b/pkg/bps/metrics.go @@ -0,0 +1,88 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "sync/atomic" + + m "github.com/ethersphere/bee/v2/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +// Counters are the per-cohort counters of the silent outcomes SWIP-74 makes a +// broker expose. +type Counters struct { + WrongStream uint64 // a frame from a subscriber stream + UnknownKind uint64 // a frame of a kind BPS-lite does not define + WrongChallenge uint64 // a frame whose challenge is not its stream's + InvalidSOC uint64 // a chunk that does not validate at the derived id + AuthTimeout uint64 // a pending stream disconnected at the auth timeout + Retransmit uint64 // a DATA frame below the cursor + QueueReset uint64 // a subscriber stream reset for a full queue +} + +type counters struct { + wrongStream atomic.Uint64 + unknownKind atomic.Uint64 + wrongChallenge atomic.Uint64 + invalidSOC atomic.Uint64 + authTimeout atomic.Uint64 + retransmit atomic.Uint64 + queueReset atomic.Uint64 +} + +func (c *counters) snapshot() Counters { + return Counters{ + WrongStream: c.wrongStream.Load(), + UnknownKind: c.unknownKind.Load(), + WrongChallenge: c.wrongChallenge.Load(), + InvalidSOC: c.invalidSOC.Load(), + AuthTimeout: c.authTimeout.Load(), + Retransmit: c.retransmit.Load(), + QueueReset: c.queueReset.Load(), + } +} + +type metrics struct { + Outcomes *prometheus.CounterVec + Delivered prometheus.Counter + Cohorts prometheus.Gauge +} + +func newMetrics() metrics { + subsystem := "bps" + + return metrics{ + Outcomes: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "dropped_frames", + Help: "Frames dropped or streams reset by the broker, by outcome.", + }, []string{"outcome"}), + Delivered: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "accepted_updates", + Help: "DATA updates accepted by the broker.", + }), + Cohorts: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "cohorts", + Help: "Live cohorts on the broker.", + }), + } +} + +// Metrics returns the prometheus collectors of the service. +func (s *Service) Metrics() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(s.metrics) +} + +// count increments the per-cohort counter c and its prometheus outcome. +func (s *Service) count(c *atomic.Uint64, outcome string) { + c.Add(1) + s.metrics.Outcomes.WithLabelValues(outcome).Inc() +} diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index c0e2696e9a3..9218a354c1c 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -22,21 +22,22 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package +// What the topic binds to. One binding. type TopicBinding int32 const ( - TopicBinding_BINDING_UNDEFINED TopicBinding = 0 - TopicBinding_BINDING_FEED TopicBinding = 1 + TopicBinding_TOPIC_BINDING_UNSPECIFIED TopicBinding = 0 + TopicBinding_FEED_TOPIC TopicBinding = 4 ) var TopicBinding_name = map[int32]string{ - 0: "BINDING_UNDEFINED", - 1: "BINDING_FEED", + 0: "TOPIC_BINDING_UNSPECIFIED", + 4: "FEED_TOPIC", } var TopicBinding_value = map[string]int32{ - "BINDING_UNDEFINED": 0, - "BINDING_FEED": 1, + "TOPIC_BINDING_UNSPECIFIED": 0, + "FEED_TOPIC": 4, } func (x TopicBinding) String() string { @@ -50,24 +51,24 @@ func (TopicBinding) EnumDescriptor() ([]byte, []int) { type Status int32 const ( - Status_STATUS_UNDEFINED Status = 0 - Status_STATUS_OK Status = 1 - Status_STATUS_FULL Status = 2 - Status_STATUS_REJECTED Status = 3 + Status_STATUS_UNSPECIFIED Status = 0 + Status_OK Status = 1 + Status_FULL Status = 2 + Status_REJECTED Status = 3 ) var Status_name = map[int32]string{ - 0: "STATUS_UNDEFINED", - 1: "STATUS_OK", - 2: "STATUS_FULL", - 3: "STATUS_REJECTED", + 0: "STATUS_UNSPECIFIED", + 1: "OK", + 2: "FULL", + 3: "REJECTED", } var Status_value = map[string]int32{ - "STATUS_UNDEFINED": 0, - "STATUS_OK": 1, - "STATUS_FULL": 2, - "STATUS_REJECTED": 3, + "STATUS_UNSPECIFIED": 0, + "OK": 1, + "FULL": 2, + "REJECTED": 3, } func (x Status) String() string { @@ -78,23 +79,54 @@ func (Status) EnumDescriptor() ([]byte, []int) { return fileDescriptor_786299925f8760d5, []int{1} } -type Broadcast struct { - // either service or broadcast - Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` +// What a chunk on this wire is. +type Kind int32 + +const ( + Kind_KIND_UNSPECIFIED Kind = 0 + Kind_DATA Kind = 1 + Kind_AUTH Kind = 2 +) + +var Kind_name = map[int32]string{ + 0: "KIND_UNSPECIFIED", + 1: "DATA", + 2: "AUTH", } -func (m *Broadcast) Reset() { *m = Broadcast{} } -func (m *Broadcast) String() string { return proto.CompactTextString(m) } -func (*Broadcast) ProtoMessage() {} -func (*Broadcast) Descriptor() ([]byte, []int) { +var Kind_value = map[string]int32{ + "KIND_UNSPECIFIED": 0, + "DATA": 1, + "AUTH": 2, +} + +func (x Kind) String() string { + return proto.EnumName(Kind_name, int32(x)) +} + +func (Kind) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{2} +} + +// The cohort's identity, keyed by its canonical serialisation. +type CohortSpec struct { + Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` + Principal []byte `protobuf:"bytes,3,opt,name=principal,proto3" json:"principal,omitempty"` +} + +func (m *CohortSpec) Reset() { *m = CohortSpec{} } +func (m *CohortSpec) String() string { return proto.CompactTextString(m) } +func (*CohortSpec) ProtoMessage() {} +func (*CohortSpec) Descriptor() ([]byte, []int) { return fileDescriptor_786299925f8760d5, []int{0} } -func (m *Broadcast) XXX_Unmarshal(b []byte) error { +func (m *CohortSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *CohortSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_Broadcast.Marshal(b, m, deterministic) + return xxx_messageInfo_CohortSpec.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -104,31 +136,43 @@ func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *Broadcast) XXX_Merge(src proto.Message) { - xxx_messageInfo_Broadcast.Merge(m, src) +func (m *CohortSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_CohortSpec.Merge(m, src) } -func (m *Broadcast) XXX_Size() int { +func (m *CohortSpec) XXX_Size() int { return m.Size() } -func (m *Broadcast) XXX_DiscardUnknown() { - xxx_messageInfo_Broadcast.DiscardUnknown(m) +func (m *CohortSpec) XXX_DiscardUnknown() { + xxx_messageInfo_CohortSpec.DiscardUnknown(m) } -var xxx_messageInfo_Broadcast proto.InternalMessageInfo +var xxx_messageInfo_CohortSpec proto.InternalMessageInfo -func (m *Broadcast) GetSoc() []byte { +func (m *CohortSpec) GetTopic() []byte { if m != nil { - return m.Soc + return m.Topic } return nil } +func (m *CohortSpec) GetBinding() TopicBinding { + if m != nil { + return m.Binding + } + return TopicBinding_TOPIC_BINDING_UNSPECIFIED +} + +func (m *CohortSpec) GetPrincipal() []byte { + if m != nil { + return m.Principal + } + return nil +} + +// Peer -> broker: the first frame on a fresh stream. type Join struct { - Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` - Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` - Principal []byte `protobuf:"bytes,3,opt,name=principal,proto3" json:"principal,omitempty"` - // ============================ - Identity []byte `protobuf:"bytes,4,opt,name=identity,proto3" json:"identity,omitempty"` + Cohort *CohortSpec `protobuf:"bytes,1,opt,name=cohort,proto3" json:"cohort,omitempty"` + Identity []byte `protobuf:"bytes,2,opt,name=identity,proto3" json:"identity,omitempty"` } func (m *Join) Reset() { *m = Join{} } @@ -164,51 +208,94 @@ func (m *Join) XXX_DiscardUnknown() { var xxx_messageInfo_Join proto.InternalMessageInfo -func (m *Join) GetTopic() []byte { +func (m *Join) GetCohort() *CohortSpec { if m != nil { - return m.Topic + return m.Cohort } return nil } -func (m *Join) GetBinding() TopicBinding { +func (m *Join) GetIdentity() []byte { if m != nil { - return m.Binding + return m.Identity } - return TopicBinding_BINDING_UNDEFINED + return nil } -func (m *Join) GetPrincipal() []byte { +// Broker -> peer, answering Join. A non-OK Ack ends the stream. +type Ack struct { + Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` + Challenge []byte `protobuf:"bytes,2,opt,name=challenge,proto3" json:"challenge,omitempty"` +} + +func (m *Ack) Reset() { *m = Ack{} } +func (m *Ack) String() string { return proto.CompactTextString(m) } +func (*Ack) ProtoMessage() {} +func (*Ack) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{2} +} +func (m *Ack) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Ack) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Ack.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Ack) XXX_Merge(src proto.Message) { + xxx_messageInfo_Ack.Merge(m, src) +} +func (m *Ack) XXX_Size() int { + return m.Size() +} +func (m *Ack) XXX_DiscardUnknown() { + xxx_messageInfo_Ack.DiscardUnknown(m) +} + +var xxx_messageInfo_Ack proto.InternalMessageInfo + +func (m *Ack) GetStatus() Status { if m != nil { - return m.Principal + return m.Status } - return nil + return Status_STATUS_UNSPECIFIED } -func (m *Join) GetIdentity() []byte { +func (m *Ack) GetChallenge() []byte { if m != nil { - return m.Identity + return m.Challenge } return nil } -type JoinAck struct { - Challenge []byte `protobuf:"bytes,1,opt,name=challenge,proto3" json:"challenge,omitempty"` - Status Status `protobuf:"varint,2,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` +// Every frame after the handshake. soc is an ordinary single-owner chunk +// whose id is derived from kind, challenge and index (see bps.ID). +type Broadcast struct { + Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` + Kind Kind `protobuf:"varint,2,opt,name=kind,proto3,enum=bps.Kind" json:"kind,omitempty"` + Challenge []byte `protobuf:"bytes,3,opt,name=challenge,proto3" json:"challenge,omitempty"` + Index uint64 `protobuf:"varint,4,opt,name=index,proto3" json:"index,omitempty"` } -func (m *JoinAck) Reset() { *m = JoinAck{} } -func (m *JoinAck) String() string { return proto.CompactTextString(m) } -func (*JoinAck) ProtoMessage() {} -func (*JoinAck) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{2} +func (m *Broadcast) Reset() { *m = Broadcast{} } +func (m *Broadcast) String() string { return proto.CompactTextString(m) } +func (*Broadcast) ProtoMessage() {} +func (*Broadcast) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{3} } -func (m *JoinAck) XXX_Unmarshal(b []byte) error { +func (m *Broadcast) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *JoinAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_JoinAck.Marshal(b, m, deterministic) + return xxx_messageInfo_Broadcast.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -218,68 +305,90 @@ func (m *JoinAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *JoinAck) XXX_Merge(src proto.Message) { - xxx_messageInfo_JoinAck.Merge(m, src) +func (m *Broadcast) XXX_Merge(src proto.Message) { + xxx_messageInfo_Broadcast.Merge(m, src) } -func (m *JoinAck) XXX_Size() int { +func (m *Broadcast) XXX_Size() int { return m.Size() } -func (m *JoinAck) XXX_DiscardUnknown() { - xxx_messageInfo_JoinAck.DiscardUnknown(m) +func (m *Broadcast) XXX_DiscardUnknown() { + xxx_messageInfo_Broadcast.DiscardUnknown(m) } -var xxx_messageInfo_JoinAck proto.InternalMessageInfo +var xxx_messageInfo_Broadcast proto.InternalMessageInfo -func (m *JoinAck) GetChallenge() []byte { +func (m *Broadcast) GetSoc() []byte { + if m != nil { + return m.Soc + } + return nil +} + +func (m *Broadcast) GetKind() Kind { + if m != nil { + return m.Kind + } + return Kind_KIND_UNSPECIFIED +} + +func (m *Broadcast) GetChallenge() []byte { if m != nil { return m.Challenge } return nil } -func (m *JoinAck) GetStatus() Status { +func (m *Broadcast) GetIndex() uint64 { if m != nil { - return m.Status + return m.Index } - return Status_STATUS_UNDEFINED + return 0 } func init() { proto.RegisterEnum("bps.TopicBinding", TopicBinding_name, TopicBinding_value) proto.RegisterEnum("bps.Status", Status_name, Status_value) - proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") + proto.RegisterEnum("bps.Kind", Kind_name, Kind_value) + proto.RegisterType((*CohortSpec)(nil), "bps.CohortSpec") proto.RegisterType((*Join)(nil), "bps.Join") - proto.RegisterType((*JoinAck)(nil), "bps.JoinAck") + proto.RegisterType((*Ack)(nil), "bps.Ack") + proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") } func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 328 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x31, 0x4f, 0xc2, 0x40, - 0x18, 0x86, 0x7b, 0x14, 0xc1, 0x7e, 0xa0, 0x1c, 0x9f, 0x98, 0x34, 0x06, 0x1b, 0x82, 0x0b, 0xc1, - 0x84, 0x41, 0x07, 0x67, 0x6a, 0x0f, 0x03, 0x92, 0x9a, 0xb4, 0x65, 0x71, 0x31, 0x6d, 0x21, 0x78, - 0x91, 0xb4, 0x17, 0x7a, 0x0e, 0xce, 0xfe, 0x01, 0x7f, 0x96, 0x23, 0xa3, 0xa3, 0x81, 0x3f, 0x62, - 0x5a, 0x8a, 0xe8, 0x76, 0xef, 0xf3, 0xde, 0xbd, 0x79, 0x92, 0x03, 0x2d, 0x10, 0x49, 0x4f, 0x2c, - 0x63, 0x19, 0xa3, 0x1a, 0x88, 0xa4, 0x7d, 0x0e, 0x9a, 0xb9, 0x8c, 0xfd, 0x69, 0xe8, 0x27, 0x12, - 0x29, 0xa8, 0x49, 0x1c, 0xea, 0xa4, 0x45, 0x3a, 0x55, 0x27, 0x3d, 0xb6, 0xdf, 0x09, 0x14, 0x47, - 0x31, 0x8f, 0xb0, 0x01, 0x07, 0x32, 0x16, 0x7c, 0x57, 0x6e, 0x03, 0x5e, 0x42, 0x39, 0xe0, 0xd1, - 0x94, 0x47, 0x73, 0xbd, 0xd0, 0x22, 0x9d, 0xe3, 0xab, 0x7a, 0x2f, 0xdd, 0xf7, 0xd2, 0xd2, 0xdc, - 0x16, 0xce, 0xee, 0x06, 0x36, 0x41, 0x13, 0x4b, 0x1e, 0x85, 0x5c, 0xf8, 0x0b, 0x5d, 0xcd, 0x66, - 0xf6, 0x00, 0xcf, 0xe0, 0x90, 0x4f, 0x67, 0x91, 0xe4, 0xf2, 0x4d, 0x2f, 0x66, 0xe5, 0x6f, 0x6e, - 0x8f, 0xa1, 0x9c, 0x4a, 0xf4, 0xc3, 0x97, 0x74, 0x24, 0x7c, 0xf6, 0x17, 0x8b, 0x59, 0x34, 0x9f, - 0xe5, 0x2e, 0x7b, 0x80, 0x17, 0x50, 0x4a, 0xa4, 0x2f, 0x5f, 0x93, 0x5c, 0xa7, 0x92, 0xe9, 0xb8, - 0x19, 0x72, 0xf2, 0xaa, 0x7b, 0x03, 0xd5, 0xbf, 0x82, 0x78, 0x0a, 0x75, 0x73, 0x68, 0x5b, 0x43, - 0xfb, 0xee, 0x69, 0x62, 0x5b, 0x6c, 0x30, 0xb4, 0x99, 0x45, 0x15, 0xa4, 0x50, 0xdd, 0xe1, 0x01, - 0x63, 0x16, 0x25, 0x5d, 0x17, 0x4a, 0xdb, 0x29, 0x6c, 0x00, 0x75, 0xbd, 0xbe, 0x37, 0x71, 0xff, - 0xbd, 0x38, 0x02, 0x2d, 0xa7, 0x0f, 0xf7, 0x94, 0x60, 0x0d, 0x2a, 0x79, 0x1c, 0x4c, 0xc6, 0x63, - 0x5a, 0xc0, 0x13, 0xa8, 0xe5, 0xc0, 0x61, 0x23, 0x76, 0xeb, 0x31, 0x8b, 0xaa, 0x66, 0xf3, 0x73, - 0x6d, 0x90, 0xd5, 0xda, 0x20, 0xdf, 0x6b, 0x83, 0x7c, 0x6c, 0x0c, 0x65, 0xb5, 0x31, 0x94, 0xaf, - 0x8d, 0xa1, 0x3c, 0x16, 0x44, 0x10, 0x94, 0xb2, 0xaf, 0xba, 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, - 0x32, 0x9a, 0x54, 0x7b, 0xb7, 0x01, 0x00, 0x00, + // 424 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0xc7, 0xe3, 0x0f, 0x4c, 0x32, 0x8d, 0xc2, 0x32, 0xaa, 0x50, 0x40, 0xad, 0x15, 0x85, 0x03, + 0x51, 0x90, 0x2a, 0x14, 0xce, 0x48, 0x38, 0xb1, 0x43, 0xdd, 0x54, 0x69, 0xb5, 0x76, 0x2e, 0x5c, + 0x22, 0x7f, 0xa9, 0x5d, 0x35, 0xec, 0xae, 0xe2, 0x45, 0x82, 0xb7, 0xe0, 0xb1, 0x38, 0xf6, 0xc8, + 0x11, 0x25, 0x2f, 0x82, 0xbc, 0x4e, 0x08, 0xe4, 0xb6, 0xf3, 0xff, 0xcf, 0xfc, 0xe6, 0x43, 0x0b, + 0xad, 0x54, 0x96, 0x17, 0x72, 0x2d, 0x94, 0x40, 0x2b, 0x95, 0x65, 0xff, 0x0b, 0xc0, 0x44, 0xdc, + 0x8b, 0xb5, 0x8a, 0x64, 0x91, 0xe1, 0x29, 0x3c, 0x51, 0x42, 0xb2, 0xac, 0x6b, 0xf4, 0x8c, 0x41, + 0x9b, 0xd6, 0x01, 0xbe, 0x85, 0xa7, 0x29, 0xe3, 0x39, 0xe3, 0x77, 0x5d, 0xb3, 0x67, 0x0c, 0x3a, + 0xa3, 0xe7, 0x17, 0x15, 0x25, 0xae, 0xcc, 0x71, 0x6d, 0xd0, 0x7d, 0x06, 0x9e, 0x41, 0x4b, 0xae, + 0x19, 0xcf, 0x98, 0x4c, 0x56, 0x5d, 0x4b, 0x63, 0x0e, 0x42, 0x7f, 0x06, 0xf6, 0x95, 0x60, 0x1c, + 0xdf, 0x80, 0x93, 0xe9, 0xb6, 0xba, 0xd3, 0xc9, 0xe8, 0x99, 0x26, 0x1e, 0x26, 0xa1, 0x3b, 0x1b, + 0x5f, 0x41, 0x93, 0xe5, 0x05, 0x57, 0x4c, 0x7d, 0xd7, 0xcd, 0xdb, 0xf4, 0x6f, 0xdc, 0xbf, 0x04, + 0xcb, 0xcb, 0x1e, 0xf0, 0x35, 0x38, 0xa5, 0x4a, 0xd4, 0xd7, 0x52, 0xb3, 0x3a, 0xa3, 0x13, 0xcd, + 0x8a, 0xb4, 0x44, 0x77, 0x56, 0x35, 0x56, 0x76, 0x9f, 0xac, 0x56, 0x05, 0xbf, 0x2b, 0x76, 0xa0, + 0x83, 0xd0, 0x97, 0xd0, 0x1a, 0xaf, 0x45, 0x92, 0x67, 0x49, 0xa9, 0x90, 0x80, 0x55, 0x8a, 0xfd, + 0x09, 0xaa, 0x27, 0x9e, 0x83, 0xfd, 0xc0, 0x78, 0xbe, 0xdb, 0xbe, 0xa5, 0xf9, 0x33, 0xc6, 0x73, + 0xaa, 0xe5, 0xff, 0xd9, 0xd6, 0x11, 0xbb, 0xba, 0x29, 0xe3, 0x79, 0xf1, 0xad, 0x6b, 0xf7, 0x8c, + 0x81, 0x4d, 0xeb, 0x60, 0xf8, 0x01, 0xda, 0xff, 0xde, 0x0f, 0xcf, 0xe1, 0x65, 0x7c, 0x73, 0x1b, + 0x4e, 0x96, 0xe3, 0x70, 0xee, 0x87, 0xf3, 0x4f, 0xcb, 0xc5, 0x3c, 0xba, 0x0d, 0x26, 0xe1, 0x34, + 0x0c, 0x7c, 0xd2, 0xc0, 0x0e, 0xc0, 0x34, 0x08, 0xfc, 0xa5, 0xce, 0x21, 0xf6, 0xf0, 0x23, 0x38, + 0xf5, 0x82, 0xf8, 0x02, 0x30, 0x8a, 0xbd, 0x78, 0x11, 0x1d, 0x55, 0x38, 0x60, 0xde, 0xcc, 0x88, + 0x81, 0x4d, 0xb0, 0xa7, 0x8b, 0xeb, 0x6b, 0x62, 0x62, 0x1b, 0x9a, 0x34, 0xb8, 0x0a, 0x26, 0x71, + 0xe0, 0x13, 0x6b, 0xf8, 0x0e, 0xec, 0x6a, 0x05, 0x3c, 0x05, 0x32, 0x0b, 0xe7, 0xfe, 0x51, 0x75, + 0x13, 0x6c, 0xdf, 0x8b, 0xbd, 0xba, 0xde, 0x5b, 0xc4, 0x97, 0xc4, 0x1c, 0x9f, 0xfd, 0xdc, 0xb8, + 0xc6, 0xe3, 0xc6, 0x35, 0x7e, 0x6f, 0x5c, 0xe3, 0xc7, 0xd6, 0x6d, 0x3c, 0x6e, 0xdd, 0xc6, 0xaf, + 0xad, 0xdb, 0xf8, 0x6c, 0xca, 0x34, 0x75, 0xf4, 0xa7, 0x7a, 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, + 0x30, 0x35, 0xa6, 0xe5, 0x61, 0x02, 0x00, 0x00, } -func (m *Broadcast) Marshal() (dAtA []byte, err error) { +func (m *CohortSpec) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -289,20 +398,32 @@ func (m *Broadcast) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Broadcast) MarshalTo(dAtA []byte) (int, error) { +func (m *CohortSpec) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *CohortSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.Soc) > 0 { - i -= len(m.Soc) - copy(dAtA[i:], m.Soc) - i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) + if len(m.Principal) > 0 { + i -= len(m.Principal) + copy(dAtA[i:], m.Principal) + i = encodeVarintBps(dAtA, i, uint64(len(m.Principal))) + i-- + dAtA[i] = 0x1a + } + if m.Binding != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x10 + } + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) i-- dAtA[i] = 0xa } @@ -334,31 +455,59 @@ func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.Identity) i = encodeVarintBps(dAtA, i, uint64(len(m.Identity))) i-- - dAtA[i] = 0x22 + dAtA[i] = 0x12 } - if len(m.Principal) > 0 { - i -= len(m.Principal) - copy(dAtA[i:], m.Principal) - i = encodeVarintBps(dAtA, i, uint64(len(m.Principal))) + if m.Cohort != nil { + { + size, err := m.Cohort.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBps(dAtA, i, uint64(size)) + } i-- - dAtA[i] = 0x1a + dAtA[i] = 0xa } - if m.Binding != 0 { - i = encodeVarintBps(dAtA, i, uint64(m.Binding)) + return len(dAtA) - i, nil +} + +func (m *Ack) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Ack) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Challenge) > 0 { + i -= len(m.Challenge) + copy(dAtA[i:], m.Challenge) + i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) i-- - dAtA[i] = 0x10 + dAtA[i] = 0x12 } - if len(m.Topic) > 0 { - i -= len(m.Topic) - copy(dAtA[i:], m.Topic) - i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) + if m.Status != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Status)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } -func (m *JoinAck) Marshal() (dAtA []byte, err error) { +func (m *Broadcast) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -368,26 +517,38 @@ func (m *JoinAck) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *JoinAck) MarshalTo(dAtA []byte) (int, error) { +func (m *Broadcast) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *JoinAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.Status != 0 { - i = encodeVarintBps(dAtA, i, uint64(m.Status)) + if m.Index != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Index)) i-- - dAtA[i] = 0x10 + dAtA[i] = 0x20 } if len(m.Challenge) > 0 { i -= len(m.Challenge) copy(dAtA[i:], m.Challenge) i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) i-- + dAtA[i] = 0x1a + } + if m.Kind != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Kind)) + i-- + dAtA[i] = 0x10 + } + if len(m.Soc) > 0 { + i -= len(m.Soc) + copy(dAtA[i:], m.Soc) + i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) + i-- dAtA[i] = 0xa } return len(dAtA) - i, nil @@ -404,13 +565,20 @@ func encodeVarintBps(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *Broadcast) Size() (n int) { +func (m *CohortSpec) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Soc) + l = len(m.Topic) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Binding != 0 { + n += 1 + sovBps(uint64(m.Binding)) + } + l = len(m.Principal) if l > 0 { n += 1 + l + sovBps(uint64(l)) } @@ -423,36 +591,52 @@ func (m *Join) Size() (n int) { } var l int _ = l - l = len(m.Topic) - if l > 0 { + if m.Cohort != nil { + l = m.Cohort.Size() n += 1 + l + sovBps(uint64(l)) } - if m.Binding != 0 { - n += 1 + sovBps(uint64(m.Binding)) - } - l = len(m.Principal) + l = len(m.Identity) if l > 0 { n += 1 + l + sovBps(uint64(l)) } - l = len(m.Identity) + return n +} + +func (m *Ack) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != 0 { + n += 1 + sovBps(uint64(m.Status)) + } + l = len(m.Challenge) if l > 0 { n += 1 + l + sovBps(uint64(l)) } return n } -func (m *JoinAck) Size() (n int) { +func (m *Broadcast) Size() (n int) { if m == nil { return 0 } var l int _ = l + l = len(m.Soc) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Kind != 0 { + n += 1 + sovBps(uint64(m.Kind)) + } l = len(m.Challenge) if l > 0 { n += 1 + l + sovBps(uint64(l)) } - if m.Status != 0 { - n += 1 + sovBps(uint64(m.Status)) + if m.Index != 0 { + n += 1 + sovBps(uint64(m.Index)) } return n } @@ -463,7 +647,7 @@ func sovBps(x uint64) (n int) { func sozBps(x uint64) (n int) { return sovBps(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *Broadcast) Unmarshal(dAtA []byte) error { +func (m *CohortSpec) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -486,15 +670,15 @@ func (m *Broadcast) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Broadcast: wiretype end group for non-group") + return fmt.Errorf("proto: CohortSpec: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Broadcast: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CohortSpec: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -521,9 +705,62 @@ func (m *Broadcast) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) - if m.Soc == nil { - m.Soc = []byte{} + m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) + if m.Topic == nil { + m.Topic = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= TopicBinding(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Principal", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Principal = append(m.Principal[:0], dAtA[iNdEx:postIndex]...) + if m.Principal == nil { + m.Principal = []byte{} } iNdEx = postIndex default: @@ -581,9 +818,9 @@ func (m *Join) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Cohort", wireType) } - var byteLen int + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowBps @@ -593,48 +830,31 @@ func (m *Join) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + if msglen < 0 { return ErrInvalidLengthBps } - postIndex := iNdEx + byteLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthBps } if postIndex > l { return io.ErrUnexpectedEOF } - m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) - if m.Topic == nil { - m.Topic = []byte{} + if m.Cohort == nil { + m.Cohort = &CohortSpec{} + } + if err := m.Cohort.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err } iNdEx = postIndex case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) - } - m.Binding = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Binding |= TopicBinding(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Principal", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -661,14 +881,86 @@ func (m *Join) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Principal = append(m.Principal[:0], dAtA[iNdEx:postIndex]...) - if m.Principal == nil { - m.Principal = []byte{} + m.Identity = append(m.Identity[:0], dAtA[iNdEx:postIndex]...) + if m.Identity == nil { + m.Identity = []byte{} } iNdEx = postIndex - case 4: + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Ack) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Ack: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Ack: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= Status(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -695,9 +987,9 @@ func (m *Join) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Identity = append(m.Identity[:0], dAtA[iNdEx:postIndex]...) - if m.Identity == nil { - m.Identity = []byte{} + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} } iNdEx = postIndex default: @@ -724,7 +1016,7 @@ func (m *Join) Unmarshal(dAtA []byte) error { } return nil } -func (m *JoinAck) Unmarshal(dAtA []byte) error { +func (m *Broadcast) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -747,13 +1039,66 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: JoinAck: wiretype end group for non-group") + return fmt.Errorf("proto: Broadcast: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: JoinAck: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Broadcast: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) + if m.Soc == nil { + m.Soc = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType) + } + m.Kind = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Kind |= Kind(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) } @@ -787,11 +1132,11 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { m.Challenge = []byte{} } iNdEx = postIndex - case 2: + case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } - m.Status = 0 + m.Index = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowBps @@ -801,7 +1146,7 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Status |= Status(b&0x7F) << shift + m.Index |= uint64(b&0x7F) << shift if b < 0x80 { break } diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index d112c17d114..a61dd2e2eab 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -2,39 +2,58 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Wire format of SWIP-74 (BPS-lite). Field and enum numbers are shared with +// SWIP-60, which extends these messages by adding fields and values only. syntax = "proto3"; package bps; option go_package = "pb"; -message Broadcast { - // either service or broadcast - bytes soc = 1; +// What the topic binds to. One binding. +enum TopicBinding { + TOPIC_BINDING_UNSPECIFIED = 0; // invalid on the wire + FEED_TOPIC = 4; // the stream is a feed on the topic } -enum TopicBinding { - BINDING_UNDEFINED = 0; - BINDING_FEED = 1; +// The cohort's identity, keyed by its canonical serialisation. +message CohortSpec { + bytes topic = 1; // 32 bytes + TopicBinding binding = 2; // FEED_TOPIC + bytes principal = 3; // 20 bytes, required: the publisher } +// Peer -> broker: the first frame on a fresh stream. message Join { - bytes topic = 1; - TopicBinding binding = 2; // which kind of topic binding - bytes principal = 3; // the "owner" - // ============================ - bytes identity = 4; // who am i + CohortSpec cohort = 1; + bytes identity = 2; // 20 bytes, required: the stream's identity } enum Status { - STATUS_UNDEFINED = 0; - STATUS_OK = 1; - STATUS_FULL = 2; - STATUS_REJECTED = 3; + STATUS_UNSPECIFIED = 0; // invalid on the wire + OK = 1; + FULL = 2; // a capacity bound + REJECTED = 3; // a spec value outside SWIP-74, or a malformed Join } -message JoinAck { - bytes challenge = 1; // the challenge to sign in order to claim the cohort's publishing role stream - Status status = 2; +// Broker -> peer, answering Join. A non-OK Ack ends the stream. +message Ack { + Status status = 1; + bytes challenge = 2; // iff OK: 32 random bytes for this stream } +// What a chunk on this wire is. +enum Kind { + KIND_UNSPECIFIED = 0; // invalid on the wire + DATA = 1; // an update of the stream's session feed + AUTH = 2; // an empty chunk: authenticates the stream, never delivered +} + +// Every frame after the handshake. soc is an ordinary single-owner chunk +// whose id is derived from kind, challenge and index (see bps.ID). +message Broadcast { + bytes soc = 1; + Kind kind = 2; + bytes challenge = 3; // 32 bytes + uint64 index = 4; +} diff --git a/pkg/node/node.go b/pkg/node/node.go index ca56e7af25f..0473a49c0ee 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -115,6 +115,7 @@ type Bee struct { pullSyncCloser io.Closer pssCloser io.Closer gsocCloser io.Closer + bpsCloser io.Closer transactionMonitorCloser io.Closer transactionCloser io.Closer listenerCloser io.Closer @@ -1107,7 +1108,8 @@ func NewBee( b.pssCloser = pssService b.gsocCloser = gsocService - bpsService := bps.New(p2ps, swarmAddress, o.FullNodeMode, logger) + bpsService := bps.New(p2ps, p2ps, o.FullNodeMode, logger, bps.DefaultOptions) + b.bpsCloser = bpsService if o.FullNodeMode { if err = p2ps.AddProtocol(bpsService.Protocol()); err != nil { return nil, fmt.Errorf("bps protocol: %w", err) @@ -1404,6 +1406,7 @@ func NewBee( if o.APIAddr != "" { // register metrics from components apiService.MustRegisterMetrics(p2ps.Metrics()...) + apiService.MustRegisterMetrics(bpsService.Metrics()...) apiService.MustRegisterMetrics(pingPong.Metrics()...) apiService.MustRegisterMetrics(acc.Metrics()...) apiService.MustRegisterMetrics(localStore.Metrics()...) @@ -1496,6 +1499,7 @@ func (b *Bee) shutdownClosers() []namedCloser { return []namedCloser{ {b.pssCloser, "pss"}, {b.gsocCloser, "gsoc"}, + {b.bpsCloser, "bps"}, {b.pusherCloser, "pusher"}, {b.pullerCloser, "puller"}, {b.accountingCloser, "accounting"}, From a3cda008185c38328e2004eecc6a4171e1c6788c Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 10 Oct 2026 04:27:29 +0200 Subject: [PATCH 14/29] chore(bps): rename and update the options Co-Authored-By: Claude Fable 5.1 --- pkg/bps/bps.go | 96 +++++++++++++++++++++++---------------------- pkg/bps/bps_test.go | 10 ++--- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index bbf45feb72b..f3beac22347 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -56,58 +56,60 @@ var ( // Options are the broker's resource bounds. Zero values take the defaults. type Options struct { - MaxSubscribers int // subscriber streams per cohort - MaxCohorts int // live cohorts per broker - MaxCohortsPerPeer int // cohorts per peer connection - MaxStreamsPerPeerCohort int // streams per peer connection per cohort - InactivityTimeout time.Duration // reclaims a cohort with no accepted frame - AuthTimeout time.Duration // disconnects a pending stream that has not authenticated - QueueSize int // outbound frames per subscriber stream - ViolationBlocklist time.Duration // blocklist duration for a protocol violation - BlocklistDuration time.Duration // blocklist duration for an auth timeout + MaxSubsPerCohort int // subscriber streams per cohort + MaxBrokerCohorts int // live cohorts per broker + MaxCohortsPerConnection int // cohorts per peer connection + MaxPeerStreamsPerCohort int // streams per peer connection per cohort + QueueCap int // outbound frames per subscriber stream + // + InactiveTimeout time.Duration // reclaims a cohort with no accepted frame + AuthWaitTimeout time.Duration // disconnects a pending stream that has not authenticated + // + ProtoBreachBlocklistDuration time.Duration // blocklist duration for a protocol violation + InvalidAuthBlocklistDuration time.Duration // blocklist duration for an auth timeout } // DefaultOptions are the SWIP-74 recommended bounds. var DefaultOptions = Options{ - MaxSubscribers: 1024, - MaxCohorts: 512, - MaxCohortsPerPeer: 16, - MaxStreamsPerPeerCohort: 2, - InactivityTimeout: 10 * time.Minute, - AuthTimeout: 30 * time.Second, - QueueSize: 64, - ViolationBlocklist: 10 * time.Minute, - BlocklistDuration: time.Minute, + MaxSubsPerCohort: 1024, + MaxBrokerCohorts: 512, + MaxCohortsPerConnection: 16, + MaxPeerStreamsPerCohort: 2, + InactiveTimeout: 10 * time.Minute, + AuthWaitTimeout: 30 * time.Second, + QueueCap: 64, + ProtoBreachBlocklistDuration: 10 * time.Minute, + InvalidAuthBlocklistDuration: time.Minute, } func (o Options) withDefaults() Options { d := DefaultOptions - if o.MaxSubscribers > 0 { - d.MaxSubscribers = o.MaxSubscribers + if o.MaxSubsPerCohort > 0 { + d.MaxSubsPerCohort = o.MaxSubsPerCohort } - if o.MaxCohorts > 0 { - d.MaxCohorts = o.MaxCohorts + if o.MaxBrokerCohorts > 0 { + d.MaxBrokerCohorts = o.MaxBrokerCohorts } - if o.MaxCohortsPerPeer > 0 { - d.MaxCohortsPerPeer = o.MaxCohortsPerPeer + if o.MaxCohortsPerConnection > 0 { + d.MaxCohortsPerConnection = o.MaxCohortsPerConnection } - if o.MaxStreamsPerPeerCohort > 0 { - d.MaxStreamsPerPeerCohort = o.MaxStreamsPerPeerCohort + if o.MaxPeerStreamsPerCohort > 0 { + d.MaxPeerStreamsPerCohort = o.MaxPeerStreamsPerCohort } - if o.InactivityTimeout > 0 { - d.InactivityTimeout = o.InactivityTimeout + if o.InactiveTimeout > 0 { + d.InactiveTimeout = o.InactiveTimeout } - if o.AuthTimeout > 0 { - d.AuthTimeout = o.AuthTimeout + if o.AuthWaitTimeout > 0 { + d.AuthWaitTimeout = o.AuthWaitTimeout } - if o.QueueSize > 0 { - d.QueueSize = o.QueueSize + if o.QueueCap > 0 { + d.QueueCap = o.QueueCap } - if o.ViolationBlocklist > 0 { - d.ViolationBlocklist = o.ViolationBlocklist + if o.ProtoBreachBlocklistDuration > 0 { + d.ProtoBreachBlocklistDuration = o.ProtoBreachBlocklistDuration } - if o.BlocklistDuration > 0 { - d.BlocklistDuration = o.BlocklistDuration + if o.InvalidAuthBlocklistDuration > 0 { + d.InvalidAuthBlocklistDuration = o.InvalidAuthBlocklistDuration } return d } @@ -457,7 +459,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } if err := s.handleFrame(co, m, &f); err != nil { if errors.Is(err, errViolation) && s.blocklister != nil { - if berr := s.blocklister.Blocklist(p.Address, s.opts.ViolationBlocklist, err.Error()); berr != nil { + if berr := s.blocklister.Blocklist(p.Address, s.opts.ProtoBreachBlocklistDuration, err.Error()); berr != nil { s.logger.Debug("blocklist failed", "peer_address", p.Address, "error", berr) } } @@ -563,10 +565,10 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort } pc := s.peers[peer.ByteString()] - if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerPeer { + if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerConnection { return nil, nil, fmt.Errorf("too many cohorts per peer: %w", errFull) } - if pc[k] >= s.opts.MaxStreamsPerPeerCohort { + if pc[k] >= s.opts.MaxPeerStreamsPerCohort { return nil, nil, fmt.Errorf("too many streams per peer per cohort: %w", errFull) } @@ -577,7 +579,7 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort co, ok := s.cohorts[k] if !ok { - if len(s.cohorts) >= s.opts.MaxCohorts { + if len(s.cohorts) >= s.opts.MaxBrokerCohorts { return nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) } co = &cohort{ @@ -586,14 +588,14 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort members: make(map[*member]struct{}), lastActivity: time.Now(), } - co.inactivity = time.AfterFunc(s.opts.InactivityTimeout, func() { s.reclaim(co) }) + co.inactivity = time.AfterFunc(s.opts.InactiveTimeout, func() { s.reclaim(co) }) s.cohorts[k] = co s.metrics.Cohorts.Set(float64(len(s.cohorts))) } co.mtx.Lock() defer co.mtx.Unlock() - if role == roleSubscriber && co.subscribers >= s.opts.MaxSubscribers { + if role == roleSubscriber && co.subscribers >= s.opts.MaxSubsPerCohort { if len(co.members) == 0 { s.removeCohort(co) } @@ -610,12 +612,12 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort m := &member{ role: role, challenge: challenge, - queue: make(chan *pb.Broadcast, s.opts.QueueSize), + queue: make(chan *pb.Broadcast, s.opts.QueueCap), } var once sync.Once m.reset = func() { once.Do(reset) } if role == rolePending { - m.authTimer = time.AfterFunc(s.opts.AuthTimeout, func() { s.authTimeout(peer, co, m) }) + m.authTimer = time.AfterFunc(s.opts.AuthWaitTimeout, func() { s.authTimeout(peer, co, m) }) } else { co.subscribers++ } @@ -676,8 +678,8 @@ func (s *Service) reclaim(co *cohort) { return } co.mtx.Lock() - if idle := time.Since(co.lastActivity); idle < s.opts.InactivityTimeout { - co.inactivity.Reset(s.opts.InactivityTimeout - idle) + if idle := time.Since(co.lastActivity); idle < s.opts.InactiveTimeout { + co.inactivity.Reset(s.opts.InactiveTimeout - idle) co.mtx.Unlock() s.mtx.Unlock() return @@ -702,7 +704,7 @@ func (s *Service) authTimeout(peer swarm.Address, co *cohort, m *member) { } s.count(&co.counters.authTimeout, "auth_timeout") if s.blocklister != nil { - if err := s.blocklister.Blocklist(peer, s.opts.BlocklistDuration, "bps auth timeout"); err != nil { + if err := s.blocklister.Blocklist(peer, s.opts.InvalidAuthBlocklistDuration, "bps auth timeout"); err != nil { s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) } } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index 4fb14fc7686..740e8be0822 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -467,7 +467,7 @@ func TestBounds(t *testing.T) { t.Run("subscribers per cohort, principal admitted outside", func(t *testing.T) { t.Parallel() - e := newEnv(t, bps.Options{MaxSubscribers: 1}) + e := newEnv(t, bps.Options{MaxSubsPerCohort: 1}) sub := e.subscribe() if _, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Identity: make([]byte, 20)}); ack.Status != pb.Status_FULL { t.Fatalf("got %s, want FULL", ack.Status) @@ -478,7 +478,7 @@ func TestBounds(t *testing.T) { t.Run("cohorts per broker", func(t *testing.T) { t.Parallel() - e := newEnv(t, bps.Options{MaxCohorts: 1}) + e := newEnv(t, bps.Options{MaxBrokerCohorts: 1}) e.subscribe() spec := e.spec() spec.Topic = make([]byte, 32) @@ -489,7 +489,7 @@ func TestBounds(t *testing.T) { t.Run("streams and cohorts per peer", func(t *testing.T) { t.Parallel() - e := newEnv(t, bps.Options{MaxStreamsPerPeerCohort: 1, MaxCohortsPerPeer: 1}) + e := newEnv(t, bps.Options{MaxPeerStreamsPerCohort: 1, MaxCohortsPerConnection: 1}) c, _ := e.client(swarm.RandAddress(t)) e.join(c, e.principal) if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: e.principal}); err == nil { @@ -503,7 +503,7 @@ func TestBounds(t *testing.T) { t.Run("auth timeout", func(t *testing.T) { t.Parallel() - e := newEnv(t, bps.Options{AuthTimeout: 50 * time.Millisecond}) + e := newEnv(t, bps.Options{AuthWaitTimeout: 50 * time.Millisecond}) e.subscribe() pub, overlay := e.publisher() expectDone(t, pub) @@ -515,7 +515,7 @@ func TestBounds(t *testing.T) { t.Run("inactivity deadline", func(t *testing.T) { t.Parallel() - e := newEnv(t, bps.Options{InactivityTimeout: 100 * time.Millisecond}) + e := newEnv(t, bps.Options{InactiveTimeout: 100 * time.Millisecond}) sub := e.subscribe() pub, _ := e.publisher() e.publish(pub, bps.KindAuth, 0, nil) From 41d7770ee782d3f216edd79ffee01494663e229a Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 10 Oct 2026 04:51:38 +0200 Subject: [PATCH 15/29] chore(bps): rename cohort.lastActivity to inactiveSince Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 790d490602e80adff6c0d5b63ac52e217125add4) --- pkg/bps/bps.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index f3beac22347..1b4cd6acfd1 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -522,12 +522,12 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { } switch { case f.Kind == pb.Kind_AUTH: - co.lastActivity = time.Now() + co.inactiveSince = time.Now() case f.Index < co.cursor: s.count(&co.counters.retransmit, "retransmit") default: co.cursor = f.Index + 1 - co.lastActivity = time.Now() + co.inactiveSince = time.Now() s.metrics.Delivered.Inc() for sub := range co.members { if sub.role != roleSubscriber { @@ -583,10 +583,10 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort return nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) } co = &cohort{ - key: k, - spec: join.Cohort, - members: make(map[*member]struct{}), - lastActivity: time.Now(), + key: k, + spec: join.Cohort, + members: make(map[*member]struct{}), + inactiveSince: time.Now(), } co.inactivity = time.AfterFunc(s.opts.InactiveTimeout, func() { s.reclaim(co) }) s.cohorts[k] = co @@ -678,7 +678,7 @@ func (s *Service) reclaim(co *cohort) { return } co.mtx.Lock() - if idle := time.Since(co.lastActivity); idle < s.opts.InactiveTimeout { + if idle := time.Since(co.inactiveSince); idle < s.opts.InactiveTimeout { co.inactivity.Reset(s.opts.InactiveTimeout - idle) co.mtx.Unlock() s.mtx.Unlock() @@ -740,11 +740,11 @@ type cohort struct { inactivity *time.Timer counters counters - mtx sync.Mutex - cursor uint64 // the lowest DATA index accepted next - lastActivity time.Time - members map[*member]struct{} - subscribers int + mtx sync.Mutex + cursor uint64 // the lowest DATA index accepted next + inactiveSince time.Time + members map[*member]struct{} + subscribers int } // shutdown returns the cohort's streams, for the caller to reset. From 7afdf2d05e83b6739377819a31bec41910d7d0e1 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 10 Oct 2026 10:27:33 +0200 Subject: [PATCH 16/29] fix(bps): refuse a frame at index 2^64-1 The cursor is set to index+1 on acceptance; the last index would wrap it to 0 and reopen the whole feed. SWIP-74 rev 9 makes that index invalid on every feed; verify now treats it as an invalid SOC, so the broker, the subscriber and the WebSocket bridge all refuse it before the signature check. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 9583d4eab3db136fc16661f84169696c78af1c58) --- pkg/bps/bps_test.go | 3 +++ pkg/bps/id.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index 740e8be0822..6a86ed7706f 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "errors" + "math" "sync" "testing" "time" @@ -544,6 +545,8 @@ func TestVerify(t *testing.T) { bps.Verify(bps.KindAuth, challenge, 9, c, e.topic, e.principal), bps.Verify(bps.KindData, challenge, 9, c, make([]byte, 32), e.principal), bps.Verify(bps.KindData, challenge, 9, c[:10], e.topic, e.principal), + bps.Verify(bps.KindData, challenge, math.MaxUint64, + e.chunk(e.signer, bps.KindData, challenge, math.MaxUint64, []byte("x")), e.topic, e.principal), } { if !errors.Is(err, bps.ErrInvalidSOC) { t.Fatalf("got %v, want invalid soc", err) diff --git a/pkg/bps/id.go b/pkg/bps/id.go index 001a373779e..880d67a4222 100644 --- a/pkg/bps/id.go +++ b/pkg/bps/id.go @@ -9,6 +9,7 @@ import ( "encoding/binary" "errors" "fmt" + "math" "github.com/ethersphere/bee/v2/pkg/bps/pb" "github.com/ethersphere/bee/v2/pkg/crypto" @@ -86,6 +87,10 @@ func verify(f *pb.Broadcast, topic, principal []byte) error { if len(f.Challenge) != ChallengeSize { return fmt.Errorf("challenge length %d: %w", len(f.Challenge), errInvalidSOC) } + // the last index is one no feed has: the cursor, set to index+1, never wraps + if f.Index == math.MaxUint64 { + return fmt.Errorf("index %d: %w", f.Index, errInvalidSOC) + } want, err := id(f.Kind, topic, f.Challenge, f.Index) if err != nil { return err From 2dfebcc3e755b48507651321ab092b71a925c3dd Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 10 Oct 2026 10:32:37 +0200 Subject: [PATCH 17/29] feat(bps): the AUTH is the only way to authenticate a stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pending stream accepts exactly one frame: its AUTH, the empty chunk at index 0 of the session's AUTH feed. Anything else from it — an update, an unknown kind — is wrong_stream, before any signature is checked. DATA is accepted on a publisher stream only, so the retransmit check no longer depends on the role and nothing upgrades a stream from the data path. An AUTH on a publisher stream is a heartbeat: it counts as activity and does nothing else. An AUTH at another index is an invalid SOC. The WebSocket bridge closes a connection that sends a DATA frame before its AUTH (4002 "expected auth") instead of letting the broker reset and blocklist the node. Tests: pending() and publisher() helpers, a frame before the AUTH, an AUTH at index 1, the heartbeat. SWIP-74 rev 10. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 90dd5635eda6f60a108be812cef0f2a3f3826aa6) --- openapi/Swarm.yaml | 15 ++++++---- pkg/api/bps.go | 9 ++++-- pkg/api/bps_test.go | 18 +++++++++++- pkg/bps/bps.go | 36 +++++++++++++---------- pkg/bps/bps_test.go | 69 +++++++++++++++++++++++++++++++++++++++----- pkg/bps/id.go | 10 +++++-- pkg/bps/pb/bps.proto | 3 +- 7 files changed, 125 insertions(+), 35 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 1947e6b5ed5..8343dbb6d42 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1118,12 +1118,15 @@ paths: `authTopic` is `keccak256("bps-service:v1" | 0x02 | topic | challenge)`. 2. Client sends binary frames `kind (1) | index (8, big-endian) | soc`, where `kind` is 1 (DATA) or 2 (AUTH) and `soc` is a single-owner chunk signed by `owner` with - `id = keccak256(sessionTopic | index)`. An AUTH chunk is empty and only authenticates - the stream. The first frame must arrive within 30 seconds. Indexes must not be reused - across sessions; frames below the broker's cursor are dropped silently. - - Close codes: 4001 invalid soc, 4002 invalid message (wrong frame type, unknown kind, - auth timeout), 4003 broker stream closed. + `id = keccak256(sessionTopic | index)`. The first frame must be the AUTH — the empty + chunk at index 0 of `authTopic`, which authenticates the stream — and must arrive within + 30 seconds; sent again later it is a heartbeat. DATA frames are updates of + `dataTopic`. Indexes must not be reused across sessions; frames below the broker's + cursor are dropped silently. + + Close codes: 4001 invalid soc (an AUTH at another index or with a payload included), + 4002 invalid message (wrong frame type, unknown kind, a frame before the AUTH, auth + timeout), 4003 broker stream closed. tags: - BPS parameters: diff --git a/pkg/api/bps.go b/pkg/api/bps.go index a846ec3bcde..565cc992d01 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -34,7 +34,7 @@ const ( bpsMaxFrameSize = bpsFrameHeader + bpsMaxSOCSize ) -// bpsAuthTimeout bounds how long a publisher may take to send its first frame. +// bpsAuthTimeout bounds how long a publisher may take to send its AUTH. var bpsAuthTimeout = 30 * time.Second var errBPSBrokerGone = errors.New("broker stream closed") @@ -321,7 +321,7 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. _ = conn.Close() }() - // the publisher declares the principal as its identity and authenticates with its first frame + // the publisher declares the principal as its identity and authenticates with its AUTH sess, err := s.bps.Join(ctx, bps.JoinRequest{ Broker: req.broker, Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, @@ -375,6 +375,11 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. s.bpsClose(conn, code, reason) return } + // the broker resets a pending stream that sends anything but its AUTH + if !authenticated && kind != bps.KindAuth { + s.bpsClose(conn, bpsCloseInvalidMessage, "expected auth") + return + } if err := sess.Publish(ctx, kind, index, chunk); err != nil { s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) return diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index d316c0cb490..c68e2c2ec75 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -453,6 +453,22 @@ func TestBPSPublishFrameErrors(t *testing.T) { }, code: api.BPSCloseInvalidSOC, }, + { + name: "auth at index 1", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 1, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 1, nil))) + }, + code: api.BPSCloseInvalidSOC, + }, + { + name: "data before auth", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) + }, + code: api.BPSCloseInvalidMessage, + }, { name: "unknown kind", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { @@ -512,7 +528,7 @@ func TestBPSPublishSessionErrors(t *testing.T) { f.sess.publishErr = errors.New("write failed") conn := f.dial(t, "publish") challenge := readChallenge(t, f, conn) - _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil))) expectClose(t, conn, api.BPSCloseBrokerGone) }) } diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 1b4cd6acfd1..15cc54acb16 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -7,10 +7,11 @@ // // A cohort is identified by its spec {topic, FEED_TOPIC, principal}. Every // peer joins with the spec and its own identity and is answered with a random -// per-stream challenge. The principal publishes ordinary single-owner chunks that -// are updates of the session feed on keccak256(topic | challenge); its first -// valid frame authenticates the stream. The broker keeps one cursor per cohort and -// delivers every accepted update to every subscriber stream. +// per-stream challenge. A stream declaring the principal is pending until its +// AUTH — the empty chunk at index 0 of the session's AUTH feed — authenticates +// it; a publisher stream then sends ordinary single-owner chunks that are updates +// of the session feed on keccak256(topic | challenge). The broker keeps one cursor +// per cohort and delivers every accepted update to every subscriber stream. package bps import ( @@ -487,8 +488,9 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { role, cursor := m.role, co.cursor co.mtx.Unlock() - // 1. only a publisher or pending stream may send - if role == roleSubscriber { + // 1. a publisher stream sends publications and heartbeats, a pending stream + // its AUTH; anything else is not what the stream may send + if role == roleSubscriber || role == rolePending && f.Kind != pb.Kind_AUTH { s.count(&co.counters.wrongStream, "wrong_stream") return fmt.Errorf("%w: %w", errViolation, errWrongStream) } @@ -502,13 +504,13 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { s.count(&co.counters.wrongChallenge, "wrong_challenge") return nil } - // 4. on a publisher stream a DATA frame below the cursor is a retransmit; - // on a pending stream it is checked after the signature, so that it authenticates - if role == rolePublisher && f.Kind == pb.Kind_DATA && f.Index < cursor { + // 4. a DATA frame below the cursor is a retransmit + if f.Kind == pb.Kind_DATA && f.Index < cursor { s.count(&co.counters.retransmit, "retransmit") return nil } - // 5. the chunk must be the principal's at the id derived from the frame + // 5. the chunk must be the principal's at the id derived from the frame; + // an AUTH is moreover the empty chunk at index 0 if err := verify(f, co.spec.Topic, co.spec.Principal); err != nil { s.count(&co.counters.invalidSOC, "invalid_soc") return fmt.Errorf("%w: %w", errViolation, err) @@ -516,14 +518,16 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { var overflow []*member co.mtx.Lock() - if m.role == rolePending { - m.role = rolePublisher - m.authTimer.Stop() - } switch { case f.Kind == pb.Kind_AUTH: + // the auth of a pending stream, a heartbeat on a publisher stream + if m.role == rolePending { + m.role = rolePublisher + m.authTimer.Stop() + } co.inactiveSince = time.Now() case f.Index < co.cursor: + // raced with another publisher stream of the principal's s.count(&co.counters.retransmit, "retransmit") default: co.cursor = f.Index + 1 @@ -730,8 +734,8 @@ type role int const ( roleSubscriber role = iota + 1 // receives, never publishes - rolePending // declared the principal as identity, not authenticated yet - rolePublisher // authenticated: may publish, receives nothing + rolePending // declared the principal as identity, no AUTH yet + rolePublisher // authenticated by its AUTH: may publish, receives nothing ) type cohort struct { diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index 6a86ed7706f..6290bebdf7a 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -101,13 +101,22 @@ func (e *env) subscribe() bps.Session { return e.join(c, addr) } -func (e *env) publisher() (bps.Session, swarm.Address) { +// pending joins as the principal and sends nothing: a pending stream. +func (e *env) pending() (bps.Session, swarm.Address) { e.t.Helper() overlay := swarm.RandAddress(e.t) c, _ := e.client(overlay) return e.join(c, e.principal), overlay } +// publisher joins as the principal and authenticates the stream with its AUTH. +func (e *env) publisher() (bps.Session, swarm.Address) { + e.t.Helper() + s, overlay := e.pending() + e.publish(s, bps.KindAuth, 0, nil) + return s, overlay +} + func (e *env) chunk(signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { e.t.Helper() id, err := bps.ID(kind, e.topic, challenge, index) @@ -196,7 +205,7 @@ func TestBroadcast(t *testing.T) { t.Fatal("want a distinct 32-byte challenge per stream") } - // AUTH authenticates and is never delivered + // the AUTH that authenticated the stream was not delivered; sent again it is a heartbeat e.publish(pub, bps.KindAuth, 0, nil) c0 := e.publish(pub, bps.KindData, 0, []byte("hello cohort")) expectMessage(t, sub, 0, c0) @@ -232,8 +241,8 @@ func TestPublisherReconnects(t *testing.T) { _ = pub1.Close() expectDone(t, pub1) - // the cohort and its cursor persist; a valid frame below the cursor - // still authenticates the new stream and is counted as a retransmit + // the cohort and its cursor persist; the new stream authenticates with its AUTH and + // an update below the cursor is counted as a retransmit pub2, _ := e.publisher() if bytes.Equal(pub1.Challenge(), pub2.Challenge()) { t.Fatal("challenge reused") @@ -322,6 +331,9 @@ func TestViolations(t *testing.T) { {"auth with payload", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { return bps.KindAuth, 0, e.chunk(e.signer, bps.KindAuth, ch, 0, []byte("x")) }}, + {"auth at index 1", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindAuth, 1, e.chunk(e.signer, bps.KindAuth, ch, 1, nil) + }}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -340,6 +352,43 @@ func TestViolations(t *testing.T) { expectNoMessage(t, sub) }) } + + // a pending stream may send nothing but its AUTH: a valid update, or a kind the + // broker does not know, is a violation before any signature is checked + for _, tc := range []struct { + name string + frame func(e *env, challenge []byte) *pb.Broadcast + }{ + {"data before auth", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ch, Index: 0, Soc: e.chunk(e.signer, bps.KindData, ch, 0, []byte("x"))} + }}, + {"unknown kind before auth", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind(7), Challenge: ch, Soc: []byte("roster")} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Identity: e.principal}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + if err := w.WriteMsg(tc.frame(e, ack.Challenge)); err != nil { + t.Fatal(err) + } + if err := r.ReadMsg(&pb.Broadcast{}); err == nil { + t.Fatal("stream not ended") + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + got := e.counters() + if got.WrongStream != 1 || got.InvalidSOC != 0 || got.UnknownKind != 0 { + t.Fatalf("counters %+v", got) + } + expectNoMessage(t, sub) + }) + } } // raw opens a stream to the broker and joins with join, returning the Ack. @@ -387,7 +436,9 @@ func TestDroppedFrames(t *testing.T) { } } - // another kind and another challenge are dropped, not punished + write(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Index: 0, Soc: e.chunk(e.signer, bps.KindAuth, ack.Challenge, 0, nil)}) + + // on a publisher stream another kind and another challenge are dropped, not punished write(&pb.Broadcast{Kind: pb.Kind(7), Challenge: ack.Challenge, Soc: []byte("roster")}) other := make([]byte, 32) write(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: other, Soc: e.chunk(e.signer, bps.KindData, other, 0, []byte("old"))}) @@ -506,7 +557,7 @@ func TestBounds(t *testing.T) { t.Parallel() e := newEnv(t, bps.Options{AuthWaitTimeout: 50 * time.Millisecond}) e.subscribe() - pub, overlay := e.publisher() + pub, overlay := e.pending() expectDone(t, pub) eventually(t, func() bool { return e.bl.blocklisted(overlay) }) if got := e.counters().AuthTimeout; got != 1 { @@ -519,7 +570,6 @@ func TestBounds(t *testing.T) { e := newEnv(t, bps.Options{InactiveTimeout: 100 * time.Millisecond}) sub := e.subscribe() pub, _ := e.publisher() - e.publish(pub, bps.KindAuth, 0, nil) expectDone(t, sub) expectDone(t, pub) if _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}); ok { @@ -538,8 +588,13 @@ func TestVerify(t *testing.T) { if err := bps.Verify(bps.KindData, challenge, 9, c, e.topic, e.principal); err != nil { t.Fatal(err) } + auth := e.chunk(e.signer, bps.KindAuth, challenge, 0, nil) + if err := bps.Verify(bps.KindAuth, challenge, 0, auth, e.topic, e.principal); err != nil { + t.Fatal(err) + } other := make([]byte, 20) for _, err := range []error{ + bps.Verify(bps.KindAuth, challenge, 1, e.chunk(e.signer, bps.KindAuth, challenge, 1, nil), e.topic, e.principal), bps.Verify(bps.KindData, challenge, 9, c, e.topic, other), bps.Verify(bps.KindData, challenge[:31], 9, c, e.topic, e.principal), bps.Verify(bps.KindAuth, challenge, 9, c, e.topic, e.principal), diff --git a/pkg/bps/id.go b/pkg/bps/id.go index 880d67a4222..54801b202b7 100644 --- a/pkg/bps/id.go +++ b/pkg/bps/id.go @@ -33,7 +33,9 @@ type Kind int const ( KindUnspecified Kind = iota KindData // an update of the stream's session feed - KindAuth // an empty chunk that authenticates the stream + // KindAuth is the empty chunk at index 0 of the session's AUTH feed: it authenticates + // a pending stream, is a heartbeat on a publisher stream, and is never delivered. + KindAuth ) func (k Kind) proto() pb.Kind { @@ -78,7 +80,7 @@ func id(kind pb.Kind, topic, challenge []byte, index uint64) ([]byte, error) { // Verify checks that the frame fields kind, challenge and index give the id of // the chunk, and that the chunk validates as the principal's single-owner chunk -// at keccak256(id | principal). An AUTH chunk must moreover be empty. +// at keccak256(id | principal). An AUTH chunk must moreover be at index 0 and empty. func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, principal []byte) error { return verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, principal) } @@ -91,6 +93,10 @@ func verify(f *pb.Broadcast, topic, principal []byte) error { if f.Index == math.MaxUint64 { return fmt.Errorf("index %d: %w", f.Index, errInvalidSOC) } + // a session has one AUTH chunk, at index 0 of its AUTH feed + if f.Kind == pb.Kind_AUTH && f.Index != 0 { + return fmt.Errorf("auth chunk at index %d: %w", f.Index, errInvalidSOC) + } want, err := id(f.Kind, topic, f.Challenge, f.Index) if err != nil { return err diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index a61dd2e2eab..06685272641 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -46,7 +46,8 @@ message Ack { enum Kind { KIND_UNSPECIFIED = 0; // invalid on the wire DATA = 1; // an update of the stream's session feed - AUTH = 2; // an empty chunk: authenticates the stream, never delivered + AUTH = 2; // the empty chunk at index 0 of the session's AUTH feed: authenticates + // a pending stream, a heartbeat on a publisher stream, never delivered } // Every frame after the handshake. soc is an ordinary single-owner chunk From 054776fde173678bb5bf03bdec5963d65e8aeec8 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 10 Oct 2026 11:22:45 +0200 Subject: [PATCH 18/29] fix(bps): the bridge checks a frame in the broker's order, and refuses the owner as a subscriber Review of the AUTH-only change found three places where the bridge could still get its own node reset or blocklisted by the broker, and one stale string: - a frame before the AUTH is refused (4002 "expected auth") before its signature is checked, as the broker's step 1 precedes its step 5; a bad DATA before the AUTH no longer reports 4001; - the bridge's 30 s auth deadline is armed before the join, so it always falls before the broker's and the broker never blocklists this node for a slow dApp; - /bps/subscribe refuses identity == owner with 400: such a stream is a pending publisher stream at the broker, receives nothing, and would be blocklisted at the auth deadline; - errWrongStream reads "frame not what the stream may send", since it now also covers a pending stream's premature frame. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 51f9682d46d5fabdf6637120f9742b2dc5b38c98) --- openapi/Swarm.yaml | 4 ++-- pkg/api/bps.go | 34 +++++++++++++++++++++------------- pkg/api/bps_test.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/bps/bps.go | 2 +- 4 files changed, 60 insertions(+), 16 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 8343dbb6d42..1d5f84caa91 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1087,7 +1087,7 @@ paths: schema: $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" required: true - description: Ethereum address identifying the joiner. + description: Ethereum address identifying the joiner. Must not be the owner, whose stream would be a pending publisher stream; publish instead. - in: query name: cursor schema: @@ -1125,7 +1125,7 @@ paths: cursor are dropped silently. Close codes: 4001 invalid soc (an AUTH at another index or with a payload included), - 4002 invalid message (wrong frame type, unknown kind, a frame before the AUTH, auth + 4002 invalid message (wrong frame type, any frame before the AUTH, unknown kind, auth timeout), 4003 broker stream closed. tags: - BPS diff --git a/pkg/api/bps.go b/pkg/api/bps.go index 565cc992d01..14e394afb77 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -5,6 +5,7 @@ package api import ( + "bytes" "context" "encoding/binary" "encoding/hex" @@ -34,7 +35,9 @@ const ( bpsMaxFrameSize = bpsFrameHeader + bpsMaxSOCSize ) -// bpsAuthTimeout bounds how long a publisher may take to send its AUTH. +// bpsAuthTimeout bounds how long a publisher may take to send its AUTH. It is +// counted from before the join, so that it runs out before the broker's auth +// deadline does and the broker never blocklists this node for a slow dApp. var bpsAuthTimeout = 30 * time.Second var errBPSBrokerGone = errors.New("broker stream closed") @@ -65,6 +68,12 @@ func (s *Service) bpsSubscribeWsHandler(w http.ResponseWriter, r *http.Request) jsonhttp.BadRequest(w, "missing identity") return } + // a stream declaring the owner is a pending publisher stream at the broker: it + // would receive nothing and be blocklisted at the auth deadline + if bytes.Equal(req.identity, req.owner.Bytes()) { + jsonhttp.BadRequest(w, "identity is the owner: use publish") + return + } conn, ok := s.bpsUpgrade(w, r, logger) if !ok { return @@ -280,9 +289,9 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo } } -// bpsParseFrame checks a publish frame kind | index | soc the same way the -// broker will, or returns a close code and reason on failure. -func bpsParseFrame(f bpsFrame, req bpsRequest, challenge []byte) (bps.Kind, uint64, []byte, int, string) { +// bpsParseFrame checks a publish frame kind | index | soc the same way, and in +// the same order, as the broker will, or returns a close code and reason. +func bpsParseFrame(f bpsFrame, req bpsRequest, challenge []byte, authenticated bool) (bps.Kind, uint64, []byte, int, string) { if f.typ != websocket.BinaryMessage { return 0, 0, nil, bpsCloseInvalidMessage, "expected binary frame" } @@ -290,6 +299,10 @@ func bpsParseFrame(f bpsFrame, req bpsRequest, challenge []byte) (bps.Kind, uint return 0, 0, nil, bpsCloseInvalidMessage, "short frame" } kind := bps.Kind(f.data[0]) + // a pending stream may send nothing but its AUTH; the broker would reset it + if !authenticated && kind != bps.KindAuth { + return 0, 0, nil, bpsCloseInvalidMessage, "expected auth" + } if kind != bps.KindData && kind != bps.KindAuth { return 0, 0, nil, bpsCloseInvalidMessage, "unknown kind" } @@ -321,6 +334,9 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. _ = conn.Close() }() + authTimer := time.NewTimer(bpsAuthTimeout) + defer authTimer.Stop() + // the publisher declares the principal as its identity and authenticates with its AUTH sess, err := s.bps.Join(ctx, bps.JoinRequest{ Broker: req.broker, @@ -361,25 +377,17 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. return } - authTimer := time.NewTimer(bpsAuthTimeout) - defer authTimer.Stop() - frames, gone := bpsReadFrames(conn, quit) authenticated := false for { select { case f := <-frames: - kind, index, chunk, code, reason := bpsParseFrame(f, req, challenge) + kind, index, chunk, code, reason := bpsParseFrame(f, req, challenge, authenticated) if code != 0 { s.bpsClose(conn, code, reason) return } - // the broker resets a pending stream that sends anything but its AUTH - if !authenticated && kind != bps.KindAuth { - s.bpsClose(conn, bpsCloseInvalidMessage, "expected auth") - return - } if err := sess.Publish(ctx, kind, index, chunk); err != nil { s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) return diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index c68e2c2ec75..b124cb1d366 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -191,6 +191,16 @@ func (f *bpsFixture) signedSOC(t *testing.T, signer crypto.Signer, kind bps.Kind return c.Data() } +// authenticate sends the session's AUTH and waits for the bridge to forward it. +func authenticate(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + auth := f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindAuth, 0, auth) +} + // frame returns a publish frame kind | index | soc. func frame(kind bps.Kind, index uint64, chunk []byte) []byte { b := []byte{byte(kind)} @@ -290,6 +300,16 @@ func TestBPSSubscribe(t *testing.T) { ) }) + t.Run("identity is the owner", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + jsonhttptest.Request(t, f.client, http.MethodGet, f.path("subscribe")+"?broker="+f.broker.String()+"&identity="+hex.EncodeToString(f.owner), http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "identity is the owner: use publish", + }), + ) + }) + t.Run("client frame closes 4002", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) @@ -410,11 +430,13 @@ func TestBPSPublishFrameErrors(t *testing.T) { for _, tc := range []struct { name string + auth bool // authenticate the socket first send func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) code int }{ { name: "wrong signer", + auth: true, send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, otherSigner, bps.KindData, challenge, 0, []byte("x")))) @@ -423,6 +445,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { }, { name: "wrong challenge", + auth: true, send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, make([]byte, 32), 0, []byte("x")))) @@ -431,6 +454,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { }, { name: "index not the signed one", + auth: true, send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 1, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) @@ -439,6 +463,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { }, { name: "auth relabelled as data", + auth: true, send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil))) @@ -469,6 +494,14 @@ func TestBPSPublishFrameErrors(t *testing.T) { }, code: api.BPSCloseInvalidMessage, }, + { + name: "invalid data before auth is still a message error", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, otherSigner, bps.KindData, challenge, 0, []byte("x")))) + }, + code: api.BPSCloseInvalidMessage, + }, { name: "unknown kind", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { @@ -499,6 +532,9 @@ func TestBPSPublishFrameErrors(t *testing.T) { f := newBPSFixture(t) conn := f.dial(t, "publish") challenge := readChallenge(t, f, conn) + if tc.auth { + authenticate(t, f, conn, challenge) + } tc.send(t, f, conn, challenge) expectClose(t, conn, tc.code) waitSessionClosed(t, f) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 15cc54acb16..0836726d8f0 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -48,7 +48,7 @@ var ( errSessionClosed = errors.New("bps: session closed") errProtocolError = errors.New("bps: protocol error") errViolation = errors.New("bps: protocol violation") - errWrongStream = errors.New("bps: frame from subscriber stream") + errWrongStream = errors.New("bps: frame not what the stream may send") errCohortReclaimed = errors.New("bps: cohort reclaimed") errFull = errors.New("bps: capacity bound") errNotPublisher = errors.New("bps: session is not a publisher") From 67602a0bbf1f4e1c3db86f112f91d4b76c16b88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Tue, 13 Oct 2026 03:26:37 +0200 Subject: [PATCH 19/29] docs(bps): say what the soc in a Broadcast is The frame carries the index-th update of the session feed: a single-owner chunk whose id is derived from kind, challenge and index. The generated doc comment follows. --- pkg/bps/pb/bps.pb.go | 4 ++-- pkg/bps/pb/bps.proto | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index 9218a354c1c..9ba6b51cdc0 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -275,8 +275,8 @@ func (m *Ack) GetChallenge() []byte { return nil } -// Every frame after the handshake. soc is an ordinary single-owner chunk -// whose id is derived from kind, challenge and index (see bps.ID). +// Every frame after the handshake. soc is the index-th update of the session feed: +// a single-owner chunk whose id is derived from kind, challenge and index (see bps.ID). type Broadcast struct { Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` Kind Kind `protobuf:"varint,2,opt,name=kind,proto3,enum=bps.Kind" json:"kind,omitempty"` diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index 06685272641..286b58f6834 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -50,8 +50,8 @@ enum Kind { // a pending stream, a heartbeat on a publisher stream, never delivered } -// Every frame after the handshake. soc is an ordinary single-owner chunk -// whose id is derived from kind, challenge and index (see bps.ID). +// Every frame after the handshake. soc is the index-th update of the session feed: +// a single-owner chunk whose id is derived from kind, challenge and index (see bps.ID). message Broadcast { bytes soc = 1; Kind kind = 2; From 20e55cb27ea338cf55bd880cf18224405a2c4962 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:27:10 +0000 Subject: [PATCH 20/29] test(bps): fuzz the Broadcast verify path Seeded with valid AUTH and DATA chunks and with malformed frames, so that verify never panics on hostile peer input. Co-authored-by: zelig <769725+zelig@users.noreply.github.com> --- pkg/bps/bps_fuzz_test.go | 87 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 pkg/bps/bps_fuzz_test.go diff --git a/pkg/bps/bps_fuzz_test.go b/pkg/bps/bps_fuzz_test.go new file mode 100644 index 00000000000..6b29965ad29 --- /dev/null +++ b/pkg/bps/bps_fuzz_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "testing" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// FuzzVerify fuzzes the peer-supplied Broadcast validation path used by BPS. +// It exercises malformed frames and valid signed chunks and asserts the verifier +// never panics on hostile input. +func FuzzVerify(f *testing.F) { + priv, err := crypto.GenerateSecp256k1Key() + if err != nil { + f.Fatal(err) + } + signer := crypto.NewDefaultSigner(priv) + principal, err := signer.EthereumAddress() + if err != nil { + f.Fatal(err) + } + + topic := make([]byte, swarm.HashSize) + copy(topic, "bps-fuzz-topic") + challenge := make([]byte, ChallengeSize) + copy(challenge, "0123456789abcdef0123456789abcdef") + + dataID, err := ID(KindData, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + payload, err := cac.New([]byte("fuzz payload")) + if err != nil { + f.Fatal(err) + } + validChunk, err := soc.New(dataID, payload).Sign(signer) + if err != nil { + f.Fatal(err) + } + + authID, err := ID(KindAuth, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + authPayload, err := cac.New(nil) + if err != nil { + f.Fatal(err) + } + authChunk, err := soc.New(authID, authPayload).Sign(signer) + if err != nil { + f.Fatal(err) + } + if len(authChunk.Data()) != swarm.HashSize+swarm.SocSignatureSize+swarm.SpanSize { + f.Fatal("auth seed not the empty auth layout") + } + + f.Add(validChunk.Data(), uint8(pb.Kind_DATA), challenge, uint64(0), topic, principal.Bytes()) + f.Add(authChunk.Data(), uint8(pb.Kind_AUTH), challenge, uint64(0), topic, principal.Bytes()) + f.Add([]byte{}, uint8(pb.Kind_DATA), make([]byte, ChallengeSize), uint64(0), topic, principal.Bytes()) + f.Add(make([]byte, swarm.HashSize), uint8(pb.Kind_DATA), []byte{}, uint64(0), topic[:31], principal.Bytes()[:19]) + + f.Fuzz(func(t *testing.T, socData []byte, kind uint8, challenge []byte, index uint64, topic []byte, principal []byte) { + if kind > uint8(pb.Kind_AUTH) { + kind = uint8(pb.Kind_KIND_UNSPECIFIED) + } + if err := verify(&pb.Broadcast{Soc: socData, Kind: pb.Kind(kind), Challenge: challenge, Index: index}, topic, principal); err != nil { + return + } + + if len(topic) != swarm.HashSize || len(principal) != crypto.AddressSize || len(challenge) != ChallengeSize { + return + } + if len(socData) >= swarm.HashSize { + if _, err := soc.FromChunk(swarm.NewChunk(swarm.ZeroAddress, socData)); err == nil { + _ = soc.Valid(swarm.NewChunk(swarm.ZeroAddress, socData)) + } + } + }) +} From 1c94dc1c4e38a127463d7328257ee1d5a0212d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Tue, 13 Oct 2026 03:34:11 +0200 Subject: [PATCH 21/29] refactor(api): one string for the bps upgrade failure --- pkg/api/bps.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/api/bps.go b/pkg/api/bps.go index 14e394afb77..b48e8ca7a7a 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -152,9 +152,10 @@ func (s *Service) bpsUpgrade(w http.ResponseWriter, r *http.Request, logger log. } conn, err := upgrader.Upgrade(w, r, nil) if err != nil { - logger.Debug("upgrade failed", "error", err) - logger.Error(nil, "upgrade failed") - jsonhttp.InternalServerError(w, "upgrade failed") + e := "upgrade failed" + logger.Debug(e, "error", err) + logger.Error(nil, e) + jsonhttp.InternalServerError(w, e) return nil, false } conn.SetReadLimit(bpsMaxFrameSize) From c411e4e7d7b8350cbcd528f41accd9590f42f1b9 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 13 Oct 2026 04:21:11 +0200 Subject: [PATCH 22/29] refactor(bps): refuse derives the Ack status from its cause refuse took any Status, OK included. It now takes the refusal's cause and answers FULL for a capacity bound and REJECTED for anything else, which is what both callers did by hand. Co-Authored-By: Claude Fable 5.1 --- pkg/bps/bps.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 0836726d8f0..68977da5768 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -403,7 +403,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er return fmt.Errorf("read join: %w", err) } if err := validateJoin(&join); err != nil { - return s.refuse(ctx, w, stream, pb.Status_REJECTED, err) + return s.refuse(ctx, w, stream, err) } // re-marshal so that the cohort key ignores fields BPS-lite does not define join.Cohort = &pb.CohortSpec{Topic: join.Cohort.Topic, Binding: join.Cohort.Binding, Principal: join.Cohort.Principal} @@ -416,11 +416,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er _ = stream.Reset() }) if err != nil { - status := pb.Status_REJECTED - if errors.Is(err, errFull) { - status = pb.Status_FULL - } - return s.refuse(ctx, w, stream, status, err) + return s.refuse(ctx, w, stream, err) } // detach on every exit, normal or not defer s.leave(p.Address, co, m) @@ -470,8 +466,14 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } } -// refuse answers a Join with a non-OK Ack, which ends the stream. -func (s *Service) refuse(ctx context.Context, w protobuf.Writer, stream p2p.Stream, status pb.Status, cause error) error { +// refuse answers a Join with a non-OK Ack and ends the stream. The status +// follows from the cause — FULL for a capacity bound, REJECTED for anything +// else — so that no caller can answer a refusal with OK. +func (s *Service) refuse(ctx context.Context, w protobuf.Writer, stream p2p.Stream, cause error) error { + status := pb.Status_REJECTED + if errors.Is(cause, errFull) { + status = pb.Status_FULL + } if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: status}); err != nil { _ = stream.Reset() return fmt.Errorf("write ack: %w", err) From 9f5442ac1c8f58764602b512c864497043f924df Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 13 Oct 2026 04:24:13 +0200 Subject: [PATCH 23/29] docs(bps): explain the fan-out and why the resets run after the lock Co-Authored-By: Claude Fable 5.1 --- pkg/bps/bps.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 68977da5768..e2dd05a93b7 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -535,6 +535,12 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { co.cursor = f.Index + 1 co.inactiveSince = time.Now() s.metrics.Delivered.Inc() + // fan-out: the accepted frame goes, unchanged, into every subscriber + // stream's queue. The send is non-blocking, so that one slow subscriber + // cannot stall the cohort; a stream whose queue is full is collected + // here and reset below, after co.mtx is released — reset ends the + // stream, whose handler then calls leave, which takes s.mtx and co.mtx, + // so resetting under the lock would deadlock. for sub := range co.members { if sub.role != roleSubscriber { continue @@ -548,6 +554,7 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { } } co.mtx.Unlock() + // the resets of the overflowed streams, outside the lock (see the fan-out above) for _, sub := range overflow { sub.reset() } From 0caa8d8653b65c190013d9c0872528fe6adc932f Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:06:18 -0600 Subject: [PATCH 24/29] feat(bps): make the broker role opt-in with bps-broker A full node no longer serves bps cohorts by default. The broker protocol is registered only when the node starts with bps-broker, which requires full-node mode. Subscribing needs no opt-in. Co-Authored-By: Claude Opus 5.5 --- cmd/bee/cmd/cmd.go | 2 ++ cmd/bee/cmd/start.go | 1 + openapi/Swarm.yaml | 2 +- packaging/bee.yaml | 2 ++ packaging/docker/env | 2 ++ packaging/homebrew-amd64/bee.yaml | 2 ++ packaging/homebrew-arm64/bee.yaml | 2 ++ packaging/scoop/bee.yaml | 2 ++ pkg/bps/bps.go | 11 ++++++----- pkg/bps/bps_test.go | 12 ++++++++++++ pkg/node/node.go | 9 +++++++-- 11 files changed, 39 insertions(+), 8 deletions(-) diff --git a/cmd/bee/cmd/cmd.go b/cmd/bee/cmd/cmd.go index c28ee2a18b7..90dda83e0bf 100644 --- a/cmd/bee/cmd/cmd.go +++ b/cmd/bee/cmd/cmd.go @@ -62,6 +62,7 @@ const ( optionNameChequebookVerification = "chequebook-verification" optionNameChequebookMinBalance = "chequebook-min-balance" optionNameFullNode = "full-node" + optionNameBpsBroker = "bps-broker" optionNameLightNodeLimit = "light-node-limit" optionNamePostageContractAddress = "postage-stamp-address" optionNamePostageContractStartBlock = "postage-stamp-start-block" @@ -365,6 +366,7 @@ func (c *command) setAllFlags(cmd *cobra.Command) { cmd.Flags().Bool(optionNameChequebookVerification, false, "reject full-node hive/handshake records that carry no chequebook address") cmd.Flags().String(optionNameChequebookMinBalance, "110000000000000000", "minimum chequebook token balance required for verification, in token small units (default 11 BZZ)") cmd.Flags().Bool(optionNameFullNode, false, "cause the node to start in full mode") + cmd.Flags().Bool(optionNameBpsBroker, false, "enable the bps broker (requires full node)") cmd.Flags().Int(optionNameLightNodeLimit, 100, "light node limit") cmd.Flags().String(optionNamePostageContractAddress, "", "postage stamp contract address") cmd.Flags().Uint64(optionNamePostageContractStartBlock, 0, "postage stamp contract start block number") diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index 12b82d6b744..4fee4dfa8be 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -325,6 +325,7 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo AutoTLSDomain: c.config.GetString(optionAutoTLSDomain), AutoTLSRegistrationEndpoint: c.config.GetString(optionAutoTLSRegistrationEndpoint), FullNodeMode: fullNode, + BpsBroker: c.config.GetBool(optionNameBpsBroker), LightNodeLimit: c.config.GetInt(optionNameLightNodeLimit), Logger: logger, MinimumGasTipCap: c.config.GetUint64(optionNameMinimumGasTipCap), diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 1d5f84caa91..02b17afaae8 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1054,7 +1054,7 @@ paths: summary: Subscribe to a BPS cohort description: | WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` - at `broker` as `identity` and sends every update as a binary frame + at `broker` as `identity` (`broker` must be a full node started with `bps-broker`) and sends every update as a binary frame `challenge (32) | index (8, big-endian) | soc`, where `soc` is the full single-owner chunk (`id | signature | span | payload`) signed by `owner` with `id = keccak256(keccak256(topic | challenge) | index)`. Updates that do not verify, diff --git a/packaging/bee.yaml b/packaging/bee.yaml index 4c91efae60d..85ab952a244 100644 --- a/packaging/bee.yaml +++ b/packaging/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/packaging/docker/env b/packaging/docker/env index 87043ca4f39..217631de5e9 100644 --- a/packaging/docker/env +++ b/packaging/docker/env @@ -31,6 +31,8 @@ # BEE_BOOTNODE=[/dnsaddr/mainnet.ethswarm.org] ## cause the node to always accept incoming connections (default false) # BEE_BOOTNODE_MODE=false +## enable the bps broker (requires full node) +# BEE_BPS_BROKER=false ## bzz token contract address # BEE_BZZ_TOKEN_ADDRESS= ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes (default 1000000) diff --git a/packaging/homebrew-amd64/bee.yaml b/packaging/homebrew-amd64/bee.yaml index f343dec9458..e53109ef03a 100644 --- a/packaging/homebrew-amd64/bee.yaml +++ b/packaging/homebrew-amd64/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/packaging/homebrew-arm64/bee.yaml b/packaging/homebrew-arm64/bee.yaml index 4bf3e8e117e..479a57db33d 100644 --- a/packaging/homebrew-arm64/bee.yaml +++ b/packaging/homebrew-arm64/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/packaging/scoop/bee.yaml b/packaging/scoop/bee.yaml index 2b5f6a94db0..ac971e029c5 100644 --- a/packaging/scoop/bee.yaml +++ b/packaging/scoop/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index e2dd05a93b7..a9823643cc7 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -115,9 +115,10 @@ func (o Options) withDefaults() Options { return d } -// Service is the bps protocol service. On a full node it is also the broker. +// Service is the bps protocol service. When broker is set it also serves +// cohorts to inbound joins; otherwise it only joins remote brokers. type Service struct { - fullNode bool + broker bool streamer p2p.Streamer blocklister p2p.Blocklister opts Options @@ -131,9 +132,9 @@ type Service struct { } // New returns a new bps Service. The blocklister may be nil. -func New(streamer p2p.Streamer, blocklister p2p.Blocklister, fullNode bool, logger log.Logger, o Options) *Service { +func New(streamer p2p.Streamer, blocklister p2p.Blocklister, broker bool, logger log.Logger, o Options) *Service { return &Service{ - fullNode: fullNode, + broker: broker, streamer: streamer, blocklister: blocklister, opts: o.withDefaults(), @@ -391,7 +392,7 @@ func validateJoin(join *pb.Join) error { // handler is the protocol handler on the broker. func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { - if !s.fullNode { + if !s.broker { _ = stream.Reset() return errNotBroker } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index 6290bebdf7a..0e1785e2a6c 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -608,3 +608,15 @@ func TestVerify(t *testing.T) { } } } + +func TestJoinNotBroker(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + notBroker := bps.New(nil, nil, false, log.Noop, bps.Options{}) + rec := streamtest.New(streamtest.WithProtocols(notBroker.Protocol()), streamtest.WithBaseAddr(swarm.RandAddress(t))) + c := bps.New(rec, nil, false, log.Noop, bps.Options{}) + _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: e.principal}) + if err == nil { + t.Fatal("join to a non-broker succeeded") + } +} diff --git a/pkg/node/node.go b/pkg/node/node.go index 0473a49c0ee..41d9e9a79d3 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -169,6 +169,7 @@ type Options struct { AutoTLSDomain string AutoTLSRegistrationEndpoint string FullNodeMode bool + BpsBroker bool LightNodeLimit int GasLimitFallback uint64 Logger log.Logger @@ -305,6 +306,10 @@ func NewBee( } }(b) + if !o.FullNodeMode && o.BpsBroker { + return nil, fmt.Errorf("bps broker is only allowed for full nodes") + } + if !o.FullNodeMode && o.ReserveCapacityDoubling != 0 { return nil, fmt.Errorf("reserve capacity doubling is only allowed for full nodes") } @@ -1108,9 +1113,9 @@ func NewBee( b.pssCloser = pssService b.gsocCloser = gsocService - bpsService := bps.New(p2ps, p2ps, o.FullNodeMode, logger, bps.DefaultOptions) + bpsService := bps.New(p2ps, p2ps, o.BpsBroker, logger, bps.DefaultOptions) b.bpsCloser = bpsService - if o.FullNodeMode { + if o.BpsBroker { if err = p2ps.AddProtocol(bpsService.Protocol()); err != nil { return nil, fmt.Errorf("bps protocol: %w", err) } From eac35932c8ee1ccab5159f313717a1facda8dd73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Tue, 20 Oct 2026 15:12:16 +0200 Subject: [PATCH 25/29] feat(bps): explicit multi-publisher mode (SWIP-87) (#5661) Co-authored-by: Claude Opus 5.5 Co-authored-by: acud <12988138+acud@users.noreply.github.com> --- openapi/Swarm.yaml | 81 ++-- pkg/api/bps.go | 144 +++++-- pkg/api/bps_test.go | 157 ++++++- pkg/api/export_test.go | 1 + pkg/bps/bps.go | 749 ++++++++++++++++++++++++++++----- pkg/bps/bps_fuzz_test.go | 90 +++- pkg/bps/bps_test.go | 22 +- pkg/bps/export_test.go | 4 + pkg/bps/flush_test.go | 419 ++++++++++++++++++ pkg/bps/id.go | 128 +++++- pkg/bps/metrics.go | 17 +- pkg/bps/multipublisher_test.go | 538 +++++++++++++++++++++++ pkg/bps/pb/bps.pb.go | 246 +++++++++-- pkg/bps/pb/bps.proto | 18 +- 14 files changed, 2376 insertions(+), 238 deletions(-) create mode 100644 pkg/bps/flush_test.go create mode 100644 pkg/bps/multipublisher_test.go diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 02b17afaae8..8858fadfe1e 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1053,14 +1053,20 @@ paths: get: summary: Subscribe to a BPS cohort description: | - WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` - at `broker` as `identity` (`broker` must be a full node started with `bps-broker`) and sends every update as a binary frame - `challenge (32) | index (8, big-endian) | soc`, where `soc` is the full single-owner - chunk (`id | signature | span | payload`) signed by `owner` with - `id = keccak256(keccak256(topic | challenge) | index)`. Updates that do not verify, - or whose index is below the subscriber's cursor, are dropped. + WebSocket. The node joins the BPS cohort `{topic, FEED_TOPIC, owner}` (SWIP-74, with + the multi-publisher mode of SWIP-87) at `broker` as `identity` and sends every + verified delivery as a binary frame + `kind (1) | owner (20) | challenge (32) | index (8, big-endian) | soc`, where `soc` is + the full single-owner chunk (`id | signature | span | payload`) signed by `owner` with + `id = keccak256(sessionTopic | index)`. `kind` is 1 (DATA): an update, by the cohort + owner or by an address on its latest roster; 4 (ROSTER): the owner's publisher set, + payload a `Roster` protobuf of 20-byte addresses; or 3 (EOS): the owner ended the + channel, after which the socket closes with 4004. `sessionTopic` is + `keccak256(topic | challenge)` for DATA and `keccak256("bps-service:v1" | kind | topic | challenge)` + otherwise. Deliveries that do not verify, by an owner the roster does not admit, or + below the subscriber's cursor are dropped. The endpoint is read-only: any client frame closes the socket with 4002. - Close codes: 4002 invalid message, 4003 broker stream closed. + Close codes: 4002 invalid message, 4003 broker stream closed, 4004 end of stream. tags: - BPS parameters: @@ -1093,7 +1099,7 @@ paths: schema: type: integer required: false - description: Lowest update index to deliver. Defaults to 0. + description: Lowest index of the owner's updates to deliver. Defaults to 0. responses: "200": description: Establishes a WebSocket subscription for incoming broadcasts @@ -1108,25 +1114,42 @@ paths: "/bps/publish/{owner}/{topic}": get: - summary: Publish to a BPS cohort as its principal + summary: Publish to a BPS cohort as its owner or as a rostered seat description: | - WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` - at `broker` declaring `owner` as its address. - - 1. Server sends `{"type":"challenge","challenge":"","broker":"","dataTopic":"","authTopic":""}`. - `dataTopic` is `keccak256(topic | challenge)`, the session feed for updates; - `authTopic` is `keccak256("bps-service:v1" | 0x02 | topic | challenge)`. - 2. Client sends binary frames `kind (1) | index (8, big-endian) | soc`, where `kind` is - 1 (DATA) or 2 (AUTH) and `soc` is a single-owner chunk signed by `owner` with - `id = keccak256(sessionTopic | index)`. The first frame must be the AUTH — the empty - chunk at index 0 of `authTopic`, which authenticates the stream — and must arrive within - 30 seconds; sent again later it is a heartbeat. DATA frames are updates of - `dataTopic`. Indexes must not be reused across sessions; frames below the broker's - cursor are dropped silently. - - Close codes: 4001 invalid soc (an AUTH at another index or with a payload included), - 4002 invalid message (wrong frame type, any frame before the AUTH, unknown kind, auth - timeout), 4003 broker stream closed. + WebSocket. The node joins the BPS cohort `{topic, FEED_TOPIC, owner}` (SWIP-74, with + the multi-publisher mode of SWIP-87) at `broker` declaring `identity` — the owner by + default, or an address the owner's roster names or will name (a seat). A seat's + stream receives from the start, like a subscriber's; once a roster naming the seat + has reached it, the server sends `{"type":"granted"}` and the seat may send its + AUTH, which upgrades the stream in place. A seat that sends anything before the + grant is closed with 4002. A seat whose address a later roster drops finds the + socket closed after that roster (4003). + + 1. Server sends `{"type":"challenge","challenge":"","broker":"","dataTopic":"","authTopic":""}`, + and to the owner also `"rosterTopic"` and `"eosTopic"`. + `dataTopic` is `keccak256(topic | challenge)`, the session feed for updates; the + others are `keccak256("bps-service:v1" | kind | topic | challenge)` with kind 0x02 + (AUTH), 0x04 (ROSTER), 0x03 (EOS). + 2. Client sends binary frames `kind (1) | index (8, big-endian) | soc`, where `soc` is + a single-owner chunk with `id = keccak256(sessionTopic | index)`, signed by + `identity` for kind 1 (DATA) and 2 (AUTH) and by `owner` for kind 4 (ROSTER) and 3 + (EOS), which only the owner may send. The first frame must be the AUTH — the empty + chunk at index 0 of `authTopic`, which authenticates the stream — and must arrive + within 25 seconds — of the connection for the owner, of the grant for a seat; + sent again later it is a heartbeat. DATA frames are updates of + `dataTopic`. A ROSTER carries a `Roster` protobuf naming every address that may + publish besides the owner, full state at each index; an EOS — the empty chunk at + index 0 of `eosTopic` — ends the channel. An owner that has a roster MUST publish + it again, at its next roster index, right after its AUTH: the broker may hold + none, after an inactivity reclaim or a restart. Indexes must not be reused across + sessions; frames below the broker's cursor are dropped silently. + 3. Server sends what the other publishers publish, and the owner's rosters — to + the owner too — as binary frames in the subscribe endpoint's format. + + Close codes: 4001 invalid soc (an AUTH or EOS at another index or with a payload + included), 4002 invalid message (wrong frame type, any frame before the AUTH, unknown + kind, a ROSTER or EOS from a seat, a roster that does not decode, a frame from a seat + before its grant, auth timeout), 4003 broker stream closed, 4004 end of stream. tags: - BPS parameters: @@ -1148,6 +1171,12 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" required: true description: Overlay address of the broker node. Must not be this node. + - in: query + name: identity + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: false + description: The address the stream publishes as. Defaults to the owner; any other may publish once the owner's roster names it. responses: "200": description: Establishes a WebSocket subscription for publishing broadcasts diff --git a/pkg/api/bps.go b/pkg/api/bps.go index b48e8ca7a7a..3d9ac59443e 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -28,17 +28,21 @@ const ( bpsCloseInvalidSOC = 4001 // a chunk that does not validate under the session challenge bpsCloseInvalidMessage = 4002 // frame violates the endpoint protocol bpsCloseBrokerGone = 4003 // the p2p stream to the broker ended + bpsCloseEndOfStream = 4004 // the owner ended the channel with an EOS bpsMaxCloseReason = 123 // websocket control frame payload limit minus the code bpsFrameHeader = 1 + 8 // publish frame header: kind | index (big-endian) bpsMaxSOCSize = swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + swarm.ChunkSize bpsMaxFrameSize = bpsFrameHeader + bpsMaxSOCSize + // delivery frame header: kind | owner | challenge | index (big-endian) + bpsDeliveryHeader = 1 + common.AddressLength + bps.ChallengeSize + 8 ) -// bpsAuthTimeout bounds how long a publisher may take to send its AUTH. It is -// counted from before the join, so that it runs out before the broker's auth -// deadline does and the broker never blocklists this node for a slow dApp. -var bpsAuthTimeout = 30 * time.Second +// bpsAuthTimeout bounds how long a publisher may take to send its AUTH. For the +// owner it is counted from before the join and is shorter than the broker's +// auth deadline, so that the broker never blocklists this node for a slow +// dApp; for a seat it is counted from the grant. +var bpsAuthTimeout = 25 * time.Second var errBPSBrokerGone = errors.New("broker stream closed") @@ -50,12 +54,41 @@ type bpsRequest struct { cursor uint64 } +// bpsGrantedMessage tells a seat's dApp that a roster naming its identity has +// reached the stream: it may now send its AUTH. +type bpsGrantedMessage struct { + Type string `json:"type"` +} + type bpsChallengeMessage struct { - Type string `json:"type"` - Challenge string `json:"challenge"` - Broker string `json:"broker"` - DataTopic string `json:"dataTopic"` - AuthTopic string `json:"authTopic"` + Type string `json:"type"` + Challenge string `json:"challenge"` + Broker string `json:"broker"` + DataTopic string `json:"dataTopic"` + AuthTopic string `json:"authTopic"` + RosterTopic string `json:"rosterTopic,omitempty"` // the owner's only + EOSTopic string `json:"eosTopic,omitempty"` // the owner's only +} + +// bpsDeliveryFrame encodes a delivery for the dApp: kind | owner | challenge | +// index | soc — what it needs to re-verify the chunk, whoever published it. +func bpsDeliveryFrame(msg bps.Message) []byte { + data := make([]byte, 0, bpsDeliveryHeader+len(msg.SOC)) + data = append(data, byte(msg.Kind)) + data = append(data, msg.Owner...) + data = append(data, msg.Challenge...) + data = binary.BigEndian.AppendUint64(data, msg.Index) + return append(data, msg.SOC...) +} + +// bpsSessionEnded closes the socket for a session whose stream ended: with +// 4004 after an EOS, 4003 otherwise. +func (s *Service) bpsSessionEnded(conn *websocket.Conn, sess bps.Session) { + if err := sess.Err(); errors.Is(err, bps.ErrEndOfStream) { + s.bpsClose(conn, bpsCloseEndOfStream, "end of stream") + return + } + s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) } func (s *Service) bpsSubscribeWsHandler(w http.ResponseWriter, r *http.Request) { @@ -88,6 +121,10 @@ func (s *Service) bpsPublishWsHandler(w http.ResponseWriter, r *http.Request) { if !ok { return } + // the owner publishes by default; a seat names its own address + if len(req.identity) == 0 { + req.identity = req.owner.Bytes() + } conn, ok := s.bpsUpgrade(w, r, logger) if !ok { return @@ -244,7 +281,7 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo Broker: req.broker, Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, Identity: req.identity, - Cursor: req.cursor, + Position: bps.Position{Cursors: map[string]uint64{string(req.owner.Bytes()): req.cursor}}, }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -259,15 +296,10 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo select { case msg, ok := <-sess.Messages(): if !ok { - s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) + s.bpsSessionEnded(conn, sess) return } - // challenge | index | soc: what the client needs to re-verify the chunk - data := make([]byte, 0, bps.ChallengeSize+8+len(msg.SOC)) - data = append(data, msg.Challenge...) - data = binary.BigEndian.AppendUint64(data, msg.Index) - data = append(data, msg.SOC...) - if err := s.bpsWrite(conn, websocket.BinaryMessage, data); err != nil { + if err := s.bpsWrite(conn, websocket.BinaryMessage, bpsDeliveryFrame(msg)); err != nil { logger.Debug("write broadcast failed", "error", err) return } @@ -275,7 +307,7 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo s.bpsClose(conn, bpsCloseInvalidMessage, "subscribe endpoint is read-only") return case <-sess.Done(): - s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) + s.bpsSessionEnded(conn, sess) return case <-gone: return @@ -304,14 +336,29 @@ func bpsParseFrame(f bpsFrame, req bpsRequest, challenge []byte, authenticated b if !authenticated && kind != bps.KindAuth { return 0, 0, nil, bpsCloseInvalidMessage, "expected auth" } - if kind != bps.KindData && kind != bps.KindAuth { + owner := req.identity + switch kind { + case bps.KindData, bps.KindAuth: + case bps.KindRoster, bps.KindEOS: + // the service feeds are the owner's alone + if !bytes.Equal(req.identity, req.owner.Bytes()) { + return 0, 0, nil, bpsCloseInvalidMessage, "roster and eos are the owner's" + } + owner = req.owner.Bytes() + default: return 0, 0, nil, bpsCloseInvalidMessage, "unknown kind" } index := binary.BigEndian.Uint64(f.data[1:bpsFrameHeader]) chunk := f.data[bpsFrameHeader:] - if err := bps.Verify(kind, challenge, index, chunk, req.topic, req.owner.Bytes()); err != nil { + if err := bps.Verify(kind, challenge, index, chunk, req.topic, owner); err != nil { return 0, 0, nil, bpsCloseInvalidSOC, "invalid soc" } + // the broker drops a roster that does not decode without a word + if kind == bps.KindRoster { + if _, err := bps.ParseRoster(chunk); err != nil { + return 0, 0, nil, bpsCloseInvalidMessage, "invalid roster" + } + } return kind, index, chunk, 0, "" } @@ -338,11 +385,15 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. authTimer := time.NewTimer(bpsAuthTimeout) defer authTimer.Stop() - // the publisher declares the principal as its identity and authenticates with its AUTH + // the publisher declares its identity — the owner's, or a seat the owner's + // roster names — and authenticates with its AUTH: the owner at once, a + // seat once a roster naming it has reached the stream, which upgrades it + // in place + owner := bytes.Equal(req.identity, req.owner.Bytes()) sess, err := s.bps.Join(ctx, bps.JoinRequest{ Broker: req.broker, Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, - Identity: req.owner.Bytes(), + Identity: req.identity, }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -351,6 +402,14 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. } defer sess.Close() + // a seat listens until it is granted; its auth deadline starts then + granted := owner + var grantedC <-chan struct{} + if !owner { + authTimer.Stop() + grantedC = sess.Granted() + } + challenge := sess.Challenge() if len(challenge) != bps.ChallengeSize { logger.Debug("invalid challenge length", "broker", req.broker, "length", len(challenge)) @@ -367,13 +426,27 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. s.bpsClose(conn, bpsCloseBrokerGone, "session topic") return } - if err := s.bpsWriteJSON(conn, bpsChallengeMessage{ + msg := bpsChallengeMessage{ Type: "challenge", Challenge: hex.EncodeToString(challenge), Broker: req.broker.String(), DataTopic: hex.EncodeToString(dataTopic), AuthTopic: hex.EncodeToString(authTopic), - }); err != nil { + } + if owner { + rosterTopic, err := bps.SessionTopic(bps.KindRoster, req.topic, challenge) + if err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, "session topic") + return + } + eosTopic, err := bps.SessionTopic(bps.KindEOS, req.topic, challenge) + if err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, "session topic") + return + } + msg.RosterTopic, msg.EOSTopic = hex.EncodeToString(rosterTopic), hex.EncodeToString(eosTopic) + } + if err := s.bpsWriteJSON(conn, msg); err != nil { logger.Debug("write challenge failed", "error", err) return } @@ -383,7 +456,19 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. for { select { + case <-grantedC: + grantedC, granted = nil, true + authTimer.Reset(bpsAuthTimeout) + if err := s.bpsWriteJSON(conn, bpsGrantedMessage{Type: "granted"}); err != nil { + logger.Debug("write granted failed", "error", err) + return + } case f := <-frames: + // a seat sends nothing before a roster names it + if !granted { + s.bpsClose(conn, bpsCloseInvalidMessage, "not granted") + return + } kind, index, chunk, code, reason := bpsParseFrame(f, req, challenge, authenticated) if code != 0 { s.bpsClose(conn, code, reason) @@ -397,13 +482,22 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. authenticated = true authTimer.Stop() } + case msg, ok := <-sess.Messages(): + if !ok { + s.bpsSessionEnded(conn, sess) + return + } + if err := s.bpsWrite(conn, websocket.BinaryMessage, bpsDeliveryFrame(msg)); err != nil { + logger.Debug("write broadcast failed", "error", err) + return + } case <-authTimer.C: if !authenticated { s.bpsClose(conn, bpsCloseInvalidMessage, "auth timeout") return } case <-sess.Done(): - s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) + s.bpsSessionEnded(conn, sess) return case <-gone: return diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index b124cb1d366..71e612a3d43 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -56,6 +56,7 @@ type published struct { type fakeSession struct { challenge []byte msgs chan bps.Message + granted chan struct{} done chan struct{} published chan published closed chan struct{} @@ -73,6 +74,7 @@ func newFakeSession() *fakeSession { return &fakeSession{ challenge: challenge, msgs: make(chan bps.Message), + granted: closedChan(), done: make(chan struct{}), published: make(chan published, 8), closed: make(chan struct{}), @@ -81,9 +83,16 @@ func newFakeSession() *fakeSession { func (f *fakeSession) Challenge() []byte { return f.challenge } func (f *fakeSession) Messages() <-chan bps.Message { return f.msgs } -func (f *fakeSession) Cursor() uint64 { return 0 } -func (f *fakeSession) Done() <-chan struct{} { return f.done } -func (f *fakeSession) Err() error { return f.err } +func (f *fakeSession) Position() bps.Position { return bps.Position{} } +func (f *fakeSession) Granted() <-chan struct{} { return f.granted } + +func closedChan() chan struct{} { + c := make(chan struct{}) + close(c) + return c +} +func (f *fakeSession) Done() <-chan struct{} { return f.done } +func (f *fakeSession) Err() error { return f.err } func (f *fakeSession) Publish(_ context.Context, kind bps.Kind, index uint64, b []byte) error { if f.publishErr != nil { @@ -157,9 +166,22 @@ func (f *bpsFixture) path(endpoint string) string { return fmt.Sprintf("/bps/%s/%s/%s", endpoint, hex.EncodeToString(f.owner), hex.EncodeToString(f.topic)) } +// dial opens the endpoint: subscribe as the fixture's identity, publish as +// the owner. func (f *bpsFixture) dial(t *testing.T, endpoint string) *websocket.Conn { t.Helper() - q := "broker=" + f.broker.String() + "&identity=" + hex.EncodeToString(f.identity) + if endpoint == "subscribe" { + return f.dialAs(t, endpoint, f.identity) + } + return f.dialAs(t, endpoint, nil) +} + +func (f *bpsFixture) dialAs(t *testing.T, endpoint string, identity []byte) *websocket.Conn { + t.Helper() + q := "broker=" + f.broker.String() + if identity != nil { + q += "&identity=" + hex.EncodeToString(identity) + } u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path(endpoint), RawQuery: q} conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) if err != nil { @@ -272,12 +294,14 @@ func TestBPSSubscribe(t *testing.T) { f := newBPSFixture(t) conn := f.dial(t, "subscribe") chunk := f.signedSOC(t, f.signer, bps.KindData, f.sess.challenge, 7, []byte("hello cohort")) - f.sess.msgs <- bps.Message{SOC: chunk, Challenge: f.sess.challenge, Index: 7} + f.sess.msgs <- bps.Message{Kind: bps.KindData, Owner: f.owner, SOC: chunk, Challenge: f.sess.challenge, Index: 7} typ, got, err := conn.ReadMessage() if err != nil { t.Fatal(err) } - want := append(append([]byte{}, f.sess.challenge...), 0, 0, 0, 0, 0, 0, 0, 7) + want := append([]byte{byte(bps.KindData)}, f.owner...) + want = append(want, f.sess.challenge...) + want = append(want, 0, 0, 0, 0, 0, 0, 0, 7) want = append(want, chunk...) if typ != websocket.BinaryMessage || !bytes.Equal(got, want) { t.Fatalf("got type %d %x, want binary %x", typ, got, want) @@ -617,3 +641,124 @@ func TestBPSPublishInvalidChallengeLength(t *testing.T) { } waitSessionClosed(t, f) } + +func TestBPSSubscribeEndOfStream(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "subscribe") + f.sess.end(bps.ErrEndOfStream) + expectClose(t, conn, api.BPSCloseEndOfStream) +} + +func TestBPSPublishForwardsDeliveries(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "publish") + _ = readChallenge(t, f, conn) + seat := make([]byte, 20) + copy(seat, "bps-test-seat") + f.sess.msgs <- bps.Message{Kind: bps.KindData, Owner: seat, SOC: []byte("soc"), Challenge: f.sess.challenge, Index: 1} + typ, got, err := conn.ReadMessage() + if err != nil { + t.Fatal(err) + } + if typ != websocket.BinaryMessage || got[0] != byte(bps.KindData) || !bytes.Equal(got[1:21], seat) { + t.Fatalf("got type %d %x", typ, got) + } +} + +func TestBPSPublishRoster(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) + authenticate(t, f, conn, challenge) + + payload, err := bps.RosterPayload([][]byte{f.identity}) + if err != nil { + t.Fatal(err) + } + ros := f.signedSOC(t, f.signer, bps.KindRoster, challenge, 0, payload) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindRoster, 0, ros)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindRoster, 0, ros) + + bad := f.signedSOC(t, f.signer, bps.KindRoster, challenge, 1, []byte{0x0a, 0x01, 0x01}) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindRoster, 1, bad)); err != nil { + t.Fatal(err) + } + expectClose(t, conn, api.BPSCloseInvalidMessage) +} + +func TestBPSPublishSeat(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.sess.granted = make(chan struct{}) + key, _ := crypto.GenerateSecp256k1Key() + seatSigner := crypto.NewDefaultSigner(key) + seatAddr, _ := seatSigner.EthereumAddress() + seat := seatAddr.Bytes() + + conn := f.dialAs(t, "publish", seat) + var m struct { + Type string `json:"type"` + Challenge string `json:"challenge"` + RosterTopic string `json:"rosterTopic"` + } + if err := conn.ReadJSON(&m); err != nil { + t.Fatal(err) + } + if m.Type != "challenge" || m.RosterTopic != "" { + t.Fatalf("unexpected challenge message %+v", m) + } + f.bps.mu.Lock() + req := f.bps.req + f.bps.mu.Unlock() + if !bytes.Equal(req.Identity, seat) { + t.Fatalf("unexpected join request %+v", req) + } + + // granted once a roster naming the seat has reached the stream + close(f.sess.granted) + if err := conn.ReadJSON(&m); err != nil || m.Type != "granted" { + t.Fatalf("want granted, got %+v %v", m, err) + } + challenge := f.sess.challenge + auth := f.signedSOC(t, seatSigner, bps.KindAuth, challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindAuth, 0, auth) + data := f.signedSOC(t, seatSigner, bps.KindData, challenge, 0, []byte("from the seat")) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, data)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindData, 0, data) + + // the roster is the owner's alone + payload, _ := bps.RosterPayload(nil) + ros := f.signedSOC(t, seatSigner, bps.KindRoster, challenge, 0, payload) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindRoster, 0, ros)); err != nil { + t.Fatal(err) + } + expectClose(t, conn, api.BPSCloseInvalidMessage) +} + +func TestBPSPublishSeatNotGranted(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.sess.granted = make(chan struct{}) + conn := f.dialAs(t, "publish", f.identity) + _ = readChallenge(t, f, conn) + // a seat listens until it is granted, and may send nothing before + f.sess.msgs <- bps.Message{Kind: bps.KindData, Owner: f.owner, SOC: []byte("soc"), Challenge: f.sess.challenge} + if typ, _, err := conn.ReadMessage(); err != nil || typ != websocket.BinaryMessage { + t.Fatalf("want a delivery, got %d %v", typ, err) + } + auth := f.signedSOC(t, f.signer, bps.KindAuth, f.sess.challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectClose(t, conn, api.BPSCloseInvalidMessage) +} diff --git a/pkg/api/export_test.go b/pkg/api/export_test.go index 7b424db67b1..22ef597c0a0 100644 --- a/pkg/api/export_test.go +++ b/pkg/api/export_test.go @@ -169,6 +169,7 @@ const ( BPSCloseInvalidSOC = bpsCloseInvalidSOC BPSCloseInvalidMessage = bpsCloseInvalidMessage BPSCloseBrokerGone = bpsCloseBrokerGone + BPSCloseEndOfStream = bpsCloseEndOfStream ) // SetBPSAuthTimeout overrides the auth timeout and returns a restore func. diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index a9823643cc7..ec396be5e0c 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -2,16 +2,19 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package bps implements BPS-lite (SWIP-74): a single-publisher live stream -// over a feed, through one broker, one hop. +// Package bps implements BPS-lite (SWIP-74) with the explicit multi-publisher +// mode of SWIP-87: a live stream over feeds, through one broker, one hop. // // A cohort is identified by its spec {topic, FEED_TOPIC, principal}. Every // peer joins with the spec and its own identity and is answered with a random -// per-stream challenge. A stream declaring the principal is pending until its -// AUTH — the empty chunk at index 0 of the session's AUTH feed — authenticates -// it; a publisher stream then sends ordinary single-owner chunks that are updates -// of the session feed on keccak256(topic | challenge). The broker keeps one cursor -// per cohort and delivers every accepted update to every subscriber stream. +// per-stream challenge. A stream declaring the principal — the admin — or an +// address on the admin's latest roster is pending until its AUTH — the empty +// chunk at index 0 of the session's AUTH feed — authenticates it; a publisher +// stream then sends ordinary single-owner chunks that are updates of its session +// feed on keccak256(topic | challenge), and the admin's publisher stream also its +// service messages: ROSTER, the publisher set, and EOS, the end of the channel. +// The broker keeps a cursor per publisher and per service kind, and delivers +// every accepted frame to every receiving stream not bound to its author. package bps import ( @@ -21,7 +24,6 @@ import ( "errors" "fmt" "sync" - "sync/atomic" "time" "github.com/ethersphere/bee/v2/pkg/bps/pb" @@ -53,6 +55,10 @@ var ( errFull = errors.New("bps: capacity bound") errNotPublisher = errors.New("bps: session is not a publisher") errInvalidDelivery = errors.New("bps: invalid delivery") + errEndOfStream = errors.New("bps: end of stream") + // ErrEndOfStream is a session's Err after the admin's EOS ended the channel. + ErrEndOfStream = errEndOfStream + errChannelEnded = errors.New("bps: channel ended") ) // Options are the broker's resource bounds. Zero values take the defaults. @@ -61,23 +67,27 @@ type Options struct { MaxBrokerCohorts int // live cohorts per broker MaxCohortsPerConnection int // cohorts per peer connection MaxPeerStreamsPerCohort int // streams per peer connection per cohort - QueueCap int // outbound frames per subscriber stream + MaxPublisherStreams int // publisher streams per address per cohort + QueueCap int // outbound frames per receiving stream // InactiveTimeout time.Duration // reclaims a cohort with no accepted frame AuthWaitTimeout time.Duration // disconnects a pending stream that has not authenticated + FlushTimeout time.Duration // bounds writing the queue of a stream being closed // ProtoBreachBlocklistDuration time.Duration // blocklist duration for a protocol violation InvalidAuthBlocklistDuration time.Duration // blocklist duration for an auth timeout } -// DefaultOptions are the SWIP-74 recommended bounds. +// DefaultOptions are the SWIP-74 and SWIP-87 recommended bounds. var DefaultOptions = Options{ MaxSubsPerCohort: 1024, MaxBrokerCohorts: 512, MaxCohortsPerConnection: 16, MaxPeerStreamsPerCohort: 2, + MaxPublisherStreams: 2, InactiveTimeout: 10 * time.Minute, AuthWaitTimeout: 30 * time.Second, + FlushTimeout: 5 * time.Second, QueueCap: 64, ProtoBreachBlocklistDuration: 10 * time.Minute, InvalidAuthBlocklistDuration: time.Minute, @@ -97,12 +107,18 @@ func (o Options) withDefaults() Options { if o.MaxPeerStreamsPerCohort > 0 { d.MaxPeerStreamsPerCohort = o.MaxPeerStreamsPerCohort } + if o.MaxPublisherStreams > 0 { + d.MaxPublisherStreams = o.MaxPublisherStreams + } if o.InactiveTimeout > 0 { d.InactiveTimeout = o.InactiveTimeout } if o.AuthWaitTimeout > 0 { d.AuthWaitTimeout = o.AuthWaitTimeout } + if o.FlushTimeout > 0 { + d.FlushTimeout = o.FlushTimeout + } if o.QueueCap > 0 { d.QueueCap = o.QueueCap } @@ -180,12 +196,35 @@ type Interface interface { Join(ctx context.Context, req JoinRequest) (Session, error) } -// Message is a verified DATA delivery: the chunk and the frame fields its id -// was derived from. +// Message is a verified delivery — a DATA update of an admitted publisher, or +// one of the admin's ROSTER and EOS — with the frame fields its id was derived +// from and the owner it validated for. type Message struct { + Kind Kind + Owner []byte // the publisher; the principal for ROSTER and EOS SOC []byte Challenge []byte Index uint64 + Roster [][]byte // ROSTER only: the publisher set besides the principal +} + +// Position is a subscriber's place on a channel, kept across streams and +// sessions: a cursor per publisher, the latest roster with its cursor, and +// whether the channel has ended. +type Position struct { + Cursors map[string]uint64 // by the publisher's 20-byte address as a string: the lowest DATA index delivered next + Roster [][]byte // the latest accepted roster + RosterCursor uint64 // the lowest ROSTER index accepted next + Ended bool // an EOS was accepted: the EOS cursor is past 0 +} + +func (p Position) clone() Position { + c := Position{Cursors: make(map[string]uint64, len(p.Cursors)), RosterCursor: p.RosterCursor, Ended: p.Ended} + for k, v := range p.Cursors { + c.Cursors[k] = v + } + c.Roster = append([][]byte(nil), p.Roster...) + return c } // Session is a single joined p2p stream to a broker. @@ -193,14 +232,23 @@ type Session interface { // Challenge is the challenge the broker issued for this stream. Every // chunk published on the session must be signed under it (see ID). Challenge() []byte - // Messages yields the DATA deliveries that verify against the cohort spec - // and the session cursor. The channel is closed when the stream ends. + // Messages yields the deliveries that verify end to end against the cohort + // spec, the roster and the session's cursors: DATA from the principal or a + // rostered publisher, and the principal's ROSTER and EOS. The channel is + // closed when the stream ends; after an EOS, with Err reporting it. Messages() <-chan Message - // Cursor is the lowest DATA index the session will deliver next. - Cursor() uint64 + // Position is the session's place on the channel, to pass to a later Join. + Position() Position + // Granted is closed once the session may authenticate: at once for the + // principal, and for any other identity once a roster naming it has been + // delivered on this stream — the cue a grantee waits for before its AUTH, + // which upgrades the stream in place. A publisher whose identity a roster + // drops finds its stream closed after that roster. + Granted() <-chan struct{} // Publish writes a chunk of the given kind and index, signed under the - // session challenge, to the broker. The broker does not reply. Only a - // session whose identity is the cohort principal may publish. + // session challenge, to the broker. The broker does not reply. Only the + // principal, or an identity the latest roster names, may publish; only the + // principal may publish ROSTER and EOS. Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error // Done is closed when the p2p stream ends. Done() <-chan struct{} @@ -210,11 +258,11 @@ type Session interface { Close() error } -// CohortSpec names a cohort: the feed of the principal on the topic. The -// topic binding is always FEED_TOPIC. +// CohortSpec names a cohort: the feeds on the topic of the principal and of +// whoever its roster names. The topic binding is always FEED_TOPIC. type CohortSpec struct { Topic []byte // 32 bytes: the feed topic - Principal []byte // 20 bytes: the publisher + Principal []byte // 20 bytes: the admin } func (c CohortSpec) proto() *pb.CohortSpec { @@ -226,7 +274,7 @@ type JoinRequest struct { Broker swarm.Address // the broker Spec CohortSpec // the cohort to join Identity []byte // 20 bytes: the joining stream's identity - Cursor uint64 // the lowest DATA index to deliver, the subscriber's cursor for the cohort + Position Position // the subscriber's place on the channel, from an earlier session } // Join joins a cohort at the broker. A peer whose join was refused or whose @@ -236,6 +284,10 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { if err := validateJoin(join); err != nil { return nil, err } + // a channel the subscriber has seen end is not rejoined + if req.Position.Ended { + return nil, errChannelEnded + } stream, err := s.streamer.NewStream(ctx, req.Broker, nil, protocolName, protocolVersion, streamName) if err != nil { return nil, fmt.Errorf("new stream: %w", err) @@ -266,11 +318,27 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { cancel: cancel, w: w, challenge: ack.Challenge, - publisher: bytes.Equal(req.Identity, req.Spec.Principal), + spec: req.Spec, + identity: req.Identity, + pos: req.Position.clone(), + granted: make(chan struct{}), rx: make(chan Message), done: make(chan struct{}), } - sess.cursor.Store(req.Cursor) + if bytes.Equal(req.Identity, req.Spec.Principal) { + sess.grant() + } + + // a broker that forwards what it should have refused is avoided for good: + // blocklisted for the longest period there is, which is forever + sess.forged = func(err error) { + if s.blocklister == nil { + return + } + if berr := s.blocklister.Blocklist(req.Broker, 0, err.Error()); berr != nil { + s.logger.Debug("blocklist broker failed", "broker", req.Broker, "error", berr) + } + } go sess.readLoop(stream, r, req.Spec) @@ -282,7 +350,7 @@ func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader, spec CohortSpec var readErr error defer func() { s.finish(readErr) - if readErr != nil && s.ctx.Err() == nil { + if readErr != nil && !errors.Is(readErr, errEndOfStream) && s.ctx.Err() == nil { _ = stream.Reset() } else { _ = stream.FullClose() @@ -299,44 +367,128 @@ func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader, spec CohortSpec readErr = err return } - // a retransmit below the cursor is skipped. The challenge is not - // compared: it is the publisher stream's, which the subscriber never sees. - if f.Kind != pb.Kind_DATA || f.Index < s.cursor.Load() { - continue - } - // re-verify end to end: a broker that forwards a chunk not valid - // for the spec is misbehaving, so the stream is ended - if err := verify(&f, spec.Topic, spec.Principal); err != nil { + msg, ok, err := s.accept(&f, spec) + if err != nil { + // a broker that forwards a chunk not valid for the spec, or by an + // owner the roster does not admit, is misbehaving: the stream is ended readErr = fmt.Errorf("%w: %w", errInvalidDelivery, err) + s.forged(readErr) return } - s.cursor.Store(f.Index + 1) + if !ok { + continue + } select { - case s.rx <- Message{SOC: f.Soc, Challenge: f.Challenge, Index: f.Index}: + case s.rx <- msg: case <-s.ctx.Done(): return } + if msg.Kind == KindEOS { + readErr = errEndOfStream + return + } } } +// accept re-verifies a delivery end to end. It returns false for a frame the +// subscriber drops: an AUTH, a kind it does not know, one below its cursor, a +// roster that does not decode. The challenge is not compared: it is the +// publisher stream's, which the subscriber never sees. +func (s *session) accept(f *pb.Broadcast, spec CohortSpec) (Message, bool, error) { + msg := Message{SOC: f.Soc, Challenge: f.Challenge, Index: f.Index} + switch f.Kind { + case pb.Kind_DATA: + owner, err := recoverOwner(f, spec.Topic) + if err != nil { + return msg, false, err + } + s.mtx.Lock() + defer s.mtx.Unlock() + if !s.admits(owner) { + return msg, false, fmt.Errorf("owner %x not rostered", owner) + } + k := string(owner) + if f.Index < s.pos.Cursors[k] { + return msg, false, nil + } + s.pos.Cursors[k] = f.Index + 1 + msg.Kind, msg.Owner = KindData, owner + return msg, true, nil + + case pb.Kind_ROSTER: + // the owner of a service chunk is forced to the principal + if err := verify(f, spec.Topic, spec.Principal); err != nil { + return msg, false, err + } + roster, ok := rosterOf(f.Soc) + if !ok { + return msg, false, nil + } + s.mtx.Lock() + defer s.mtx.Unlock() + if f.Index >= s.pos.RosterCursor { + s.pos.RosterCursor = f.Index + 1 + s.pos.Roster = roster + msg.Kind, msg.Owner, msg.Roster = KindRoster, spec.Principal, roster + } + // the cue: a roster naming the identity, delivered on this stream, + // while the latest roster still names it — the broker holds it too + if contains(roster, s.identity) && s.admits(s.identity) { + s.grant() + } + return msg, msg.Kind == KindRoster, nil + + case pb.Kind_EOS: + if err := verify(f, spec.Topic, spec.Principal); err != nil { + return msg, false, err + } + s.mtx.Lock() + s.pos.Ended = true + s.mtx.Unlock() + msg.Kind, msg.Owner = KindEOS, spec.Principal + return msg, true, nil + } + return msg, false, nil +} + +// admits reports whether owner may publish: the principal, or an address on the +// latest roster. Must be called with s.mtx held. +func (s *session) admits(owner []byte) bool { + return bytes.Equal(owner, s.spec.Principal) || contains(s.pos.Roster, owner) +} + +func contains(set [][]byte, a []byte) bool { + for _, b := range set { + if bytes.Equal(a, b) { + return true + } + } + return false +} + // session is the concrete Session implementation. type session struct { - ctx context.Context - cancel context.CancelFunc - wmtx sync.Mutex - w protobuf.Writer - challenge []byte - publisher bool // the identity is the cohort principal - cursor atomic.Uint64 - rx chan Message - done chan struct{} - err error - once sync.Once + ctx context.Context + cancel context.CancelFunc + wmtx sync.Mutex + w protobuf.Writer + challenge []byte + spec CohortSpec + identity []byte + forged func(error) // called on a delivery the broker should have refused + mtx sync.Mutex + pos Position + granted chan struct{} + grantedOnce sync.Once + rx chan Message + done chan struct{} + err error + once sync.Once } func (s *session) Challenge() []byte { return s.challenge } func (s *session) Messages() <-chan Message { return s.rx } -func (s *session) Cursor() uint64 { return s.cursor.Load() } +func (s *session) Granted() <-chan struct{} { return s.granted } func (s *session) Done() <-chan struct{} { return s.done } func (s *session) Err() error { return s.err } func (s *session) Close() error { @@ -344,8 +496,25 @@ func (s *session) Close() error { return nil } +func (s *session) Position() Position { + s.mtx.Lock() + defer s.mtx.Unlock() + return s.pos.clone() +} + +func (s *session) grant() { s.grantedOnce.Do(func() { close(s.granted) }) } + func (s *session) Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error { - if !s.publisher { + principal := bytes.Equal(s.identity, s.spec.Principal) + select { + case <-s.granted: + default: + return errNotPublisher + } + s.mtx.Lock() + seated := s.admits(s.identity) + s.mtx.Unlock() + if !seated || !principal && (kind == KindRoster || kind == KindEOS) { return errNotPublisher } select { @@ -359,6 +528,8 @@ func (s *session) Publish(ctx context.Context, kind Kind, index uint64, soc []by if err := s.w.WriteMsgWithContext(ctx, &f); err != nil { return fmt.Errorf("write broadcast: %w", err) } + // a roster or an EOS the admin publishes is delivered back to it, and it + // admits by it from where it lands in its stream return nil } @@ -415,6 +586,10 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er co, m, err := s.join(p.Address, &join, func() { cancel() _ = stream.Reset() + }, func() { + // closing, not resetting, lets the frames already written arrive + _ = stream.Close() + cancel() }) if err != nil { return s.refuse(ctx, w, stream, err) @@ -427,7 +602,8 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er return fmt.Errorf("write ack: %w", err) } - // the writer side: deliveries to a subscriber stream + // the writer side: deliveries to a receiving stream, and the roster a + // pending stream is named in writerDone := make(chan struct{}) go func() { defer close(writerDone) @@ -438,6 +614,24 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er m.reset() return } + case <-m.ended: + // an EOS or a revocation: write what is queued — the EOS or the + // reduced roster last — then close; a peer that does not read + // within the flush deadline is reset + flushCtx, flushCancel := context.WithTimeout(ctx, s.opts.FlushTimeout) + defer flushCancel() + for { + select { + case f := <-m.queue: + if err := w.WriteMsgWithContext(flushCtx, f); err != nil { + m.reset() + return + } + default: + m.close() + return + } + } case <-ctx.Done(): return } @@ -485,20 +679,47 @@ func (s *Service) refuse(ctx context.Context, w protobuf.Writer, stream p2p.Stre } // handleFrame validates a Broadcast arriving at the broker per SWIP-74 and -// accepts, drops or punishes it. A returned error resets the stream. +// SWIP-87 and accepts, drops or punishes it. A returned error resets the stream. func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { + principal := co.spec.Principal + admin := bytes.Equal(m.identity, principal) + co.mtx.Lock() - role, cursor := m.role, co.cursor + role := m.role + named, cued := co.rostered(m.identity), m.cued + var cursor uint64 + switch f.Kind { + case pb.Kind_DATA: + cursor = co.cursors[string(m.identity)] + case pb.Kind_ROSTER: + cursor = co.rosterCursor + } co.mtx.Unlock() - // 1. a publisher stream sends publications and heartbeats, a pending stream - // its AUTH; anything else is not what the stream may send - if role == roleSubscriber || role == rolePending && f.Kind != pb.Kind_AUTH { + // 0. a stream being closed — revoked, or ended by an EOS — is not read: what + // it still sends was sent before it read the roster or the EOS + if role == roleRevoked || role == roleEnded { + s.dropClosing(co, role) + return nil + } + // an AUTH from a spectator a roster had named on this stream, which a later + // roster no longer names, raced its revocation: the stream is closed, the + // AUTH dropped, nothing punished + if role == roleSpectator && f.Kind == pb.Kind_AUTH && !named && cued { + s.revokeSpectator(co, m) + return nil + } + // 1. a publisher stream sends publications and heartbeats — the admin's also + // its service messages — a pending stream its AUTH, a spectator stream its + // AUTH once a roster names it; anything else is not what the stream may send + if !mayEmit(role, admin, named, f.Kind) { s.count(&co.counters.wrongStream, "wrong_stream") return fmt.Errorf("%w: %w", errViolation, errWrongStream) } - // 2. a kind BPS-lite does not define is dropped - if f.Kind != pb.Kind_DATA && f.Kind != pb.Kind_AUTH { + // 2. a kind this SWIP does not define is dropped + switch f.Kind { + case pb.Kind_DATA, pb.Kind_AUTH, pb.Kind_EOS, pb.Kind_ROSTER: + default: s.count(&co.counters.unknownKind, "unknown_kind") return nil } @@ -507,65 +728,170 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { s.count(&co.counters.wrongChallenge, "wrong_challenge") return nil } - // 4. a DATA frame below the cursor is a retransmit - if f.Kind == pb.Kind_DATA && f.Index < cursor { + // 4. a DATA or ROSTER frame below its feed's cursor is a retransmit + if (f.Kind == pb.Kind_DATA || f.Kind == pb.Kind_ROSTER) && f.Index < cursor { s.count(&co.counters.retransmit, "retransmit") return nil } - // 5. the chunk must be the principal's at the id derived from the frame; - // an AUTH is moreover the empty chunk at index 0 - if err := verify(f, co.spec.Topic, co.spec.Principal); err != nil { + // 5. the chunk must be its owner's at the id derived from the frame: the + // stream's identity for DATA and AUTH, the principal for the service kinds; + // an AUTH or EOS is moreover the empty chunk at index 0 + owner := m.identity + if service(f.Kind) { + owner = principal + } + if err := verify(f, co.spec.Topic, owner); err != nil { s.count(&co.counters.invalidSOC, "invalid_soc") return fmt.Errorf("%w: %w", errViolation, err) } + // 6. a ROSTER must decode; one that does not changes nothing + var roster [][]byte + if f.Kind == pb.Kind_ROSTER { + var ok bool + if roster, ok = rosterOf(f.Soc); !ok { + s.count(&co.counters.invalidRoster, "invalid_roster") + return nil + } + } - var overflow []*member + var resets, evicted, closes []*member co.mtx.Lock() - switch { - case f.Kind == pb.Kind_AUTH: - // the auth of a pending stream, a heartbeat on a publisher stream - if m.role == rolePending { - m.role = rolePublisher - m.authTimer.Stop() + // a roster revoking the identity, or an EOS, was accepted since the checks + if r := m.role; r == roleRevoked || r == roleEnded { + co.mtx.Unlock() + s.dropClosing(co, r) + return nil + } + switch f.Kind { + case pb.Kind_AUTH: + switch { + case m.role == roleSpectator && !co.rostered(m.identity): + // revoked since the checks + co.mtx.Unlock() + s.revokeSpectator(co, m) + return nil + case m.role != rolePublisher: + // the auth of a pending stream or of a spectator a roster names + resets, evicted = co.upgrade(m, s.opts.MaxPublisherStreams) } + // on a publisher stream, a heartbeat co.inactiveSince = time.Now() - case f.Index < co.cursor: - // raced with another publisher stream of the principal's - s.count(&co.counters.retransmit, "retransmit") - default: - co.cursor = f.Index + 1 + + case pb.Kind_DATA: + k := string(m.identity) + if f.Index < co.cursors[k] { + // raced with another publisher stream of the same identity + s.count(&co.counters.retransmit, "retransmit") + break + } + co.cursors[k] = f.Index + 1 co.inactiveSince = time.Now() s.metrics.Delivered.Inc() - // fan-out: the accepted frame goes, unchanged, into every subscriber - // stream's queue. The send is non-blocking, so that one slow subscriber - // cannot stall the cohort; a stream whose queue is full is collected - // here and reset below, after co.mtx is released — reset ends the - // stream, whose handler then calls leave, which takes s.mtx and co.mtx, - // so resetting under the lock would deadlock. - for sub := range co.members { - if sub.role != roleSubscriber { - continue + resets = co.deliver(f, m) + + case pb.Kind_ROSTER: + if f.Index < co.rosterCursor { + s.count(&co.counters.retransmit, "retransmit") + break + } + co.rosterCursor = f.Index + 1 + co.inactiveSince = time.Now() + resets, closes = co.applyRoster(f, roster, m) + + case pb.Kind_EOS: + // the channel is ended: every stream is told — pending ones too — and + // closed, and nothing more is read from any of them + co.ended = true + resets = co.deliver(f, m) + for o := range co.members { + if o.role == rolePending && !o.enqueue(f) { + resets = append(resets, o) } - select { - case sub.queue <- f: - default: - s.count(&co.counters.queueReset, "queue_reset") - overflow = append(overflow, sub) + } + closes = co.end() + for _, o := range closes { + if o.authTimer != nil { + o.authTimer.Stop() } + o.role = roleEnded } } co.mtx.Unlock() - // the resets of the overflowed streams, outside the lock (see the fan-out above) - for _, sub := range overflow { - sub.reset() + + // the resets of the overflowed streams, outside the lock (see deliver) + for _, r := range resets { + s.count(&co.counters.queueReset, "queue_reset") + r.reset() + } + for _, r := range evicted { + r.reset() + } + if f.Kind == pb.Kind_EOS { + s.endCohort(co, closes) + return nil + } + // the revoked streams are closed once the roster is written to them + for _, r := range closes { + r.end() } return nil } +// mayEmit reports whether a stream of the role may send a frame of the kind +// (step 1); named is whether the latest roster names the stream's identity. +// Kinds the broker does not define are left to step 2. +func mayEmit(r role, admin, named bool, kind pb.Kind) bool { + switch r { + case rolePublisher: + return !service(kind) || admin + case rolePending: + return kind == pb.Kind_AUTH + case roleSpectator: + return named && kind == pb.Kind_AUTH + } + return false +} + +// dropClosing counts a frame from a stream being closed, which is not read. +func (s *Service) dropClosing(co *cohort, r role) { + if r == roleEnded { + s.count(&co.counters.ended, "ended") + return + } + s.count(&co.counters.revoked, "revoked") +} + +// revokeSpectator closes a spectator whose AUTH raced the roster revoking it. +func (s *Service) revokeSpectator(co *cohort, m *member) { + co.mtx.Lock() + if m.role == roleSpectator { + m.role = roleRevoked + co.spectators-- + } + co.mtx.Unlock() + s.count(&co.counters.revoked, "revoked") + m.end() +} + +// endCohort forgets a cohort ended by its admin's EOS and closes its streams +// once their queues — the EOS last — are written. +func (s *Service) endCohort(co *cohort, ms []*member) { + s.mtx.Lock() + if s.cohorts[co.key] == co { + co.mtx.Lock() + s.removeCohort(co) + co.mtx.Unlock() + } + s.mtx.Unlock() + for _, m := range ms { + m.end() + } +} + // join attaches a stream to the cohort named by the spec, creating the // cohort if no live one has it. An error wrapping errFull means a capacity -// bound was hit. -func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort, *member, error) { +// bound was hit. reset ends the stream abruptly, close after its queue. +func (s *Service) join(peer swarm.Address, join *pb.Join, reset, closeFn func()) (*cohort, *member, error) { key, err := join.Cohort.Marshal() if err != nil { return nil, nil, fmt.Errorf("marshal cohort spec: %w", err) @@ -586,12 +912,17 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort return nil, nil, fmt.Errorf("too many streams per peer per cohort: %w", errFull) } - role := roleSubscriber - if bytes.Equal(join.Identity, join.Cohort.Principal) { - role = rolePending - } - co, ok := s.cohorts[k] + if ok { + // a cohort its admin's EOS ended is being forgotten: the Join creates + // a fresh one + co.mtx.Lock() + if co.ended { + s.removeCohort(co) + ok = false + } + co.mtx.Unlock() + } if !ok { if len(s.cohorts) >= s.opts.MaxBrokerCohorts { return nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) @@ -600,6 +931,7 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort key: k, spec: join.Cohort, members: make(map[*member]struct{}), + cursors: make(map[string]uint64), inactiveSince: time.Now(), } co.inactivity = time.AfterFunc(s.opts.InactiveTimeout, func() { s.reclaim(co) }) @@ -609,7 +941,18 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort co.mtx.Lock() defer co.mtx.Unlock() - if role == roleSubscriber && co.subscribers >= s.opts.MaxSubsPerCohort { + + // the admin's identity is pending, outside the fan-out bound; any other is + // a spectator, within it — except that a rostered identity finding the set + // full is pending too: it receives the roster that names it and nothing + // else until its AUTH, so the bound is not bypassed by declaring a public + // address + role := roleSpectator + full := co.spectators >= s.opts.MaxSubsPerCohort + if bytes.Equal(join.Identity, join.Cohort.Principal) || full && co.rostered(join.Identity) { + role = rolePending + } + if role == roleSpectator && full { if len(co.members) == 0 { s.removeCohort(co) } @@ -625,15 +968,37 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort } m := &member{ role: role, + identity: join.Identity, challenge: challenge, queue: make(chan *pb.Broadcast, s.opts.QueueCap), + ended: make(chan struct{}), } - var once sync.Once + var once, endOnce sync.Once m.reset = func() { once.Do(reset) } + m.close = func() { once.Do(closeFn) } + // a stream being closed has the flush deadline to take its queue, the write + // in progress included; then it is reset, unless it closed in time + m.end = func() { + endOnce.Do(func() { + close(m.ended) + time.AfterFunc(s.opts.FlushTimeout, m.reset) + }) + } if role == rolePending { m.authTimer = time.AfterFunc(s.opts.AuthWaitTimeout, func() { s.authTimeout(peer, co, m) }) + // a rostered identity receives the roster that names it at once: its cue + if !bytes.Equal(join.Identity, join.Cohort.Principal) { + m.queue <- co.rosterFrame + m.given = co.rosterFrame + } } else { - co.subscribers++ + co.spectators++ + // the latest roster is the first delivery on a receiving stream; one + // naming the identity is the cue for its AUTH + if co.rosterFrame != nil { + m.queue <- co.rosterFrame + m.cued = co.rostered(join.Identity) + } } co.members[m] = struct{}{} @@ -668,8 +1033,8 @@ func (s *Service) leave(peer swarm.Address, co *cohort, m *member) { if m.authTimer != nil { m.authTimer.Stop() } - if m.role == roleSubscriber { - co.subscribers-- + if m.role == roleSpectator { + co.spectators-- } if len(co.members) == 0 && s.cohorts[co.key] == co { s.removeCohort(co) @@ -712,12 +1077,18 @@ func (s *Service) reclaim(co *cohort) { func (s *Service) authTimeout(peer swarm.Address, co *cohort, m *member) { co.mtx.Lock() pending := m.role == rolePending + if pending { + // an AUTH that arrives now is not accepted + m.role = roleRevoked + } co.mtx.Unlock() if !pending { return } s.count(&co.counters.authTimeout, "auth_timeout") - if s.blocklister != nil { + // a rostered identity admitted past a full fan-out set that does not + // authenticate has cost what a FULL would have: no blocklist + if s.blocklister != nil && bytes.Equal(m.identity, co.spec.Principal) { if err := s.blocklister.Blocklist(peer, s.opts.InvalidAuthBlocklistDuration, "bps auth timeout"); err != nil { s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) } @@ -743,9 +1114,11 @@ func (s *Service) Counters(spec CohortSpec) (Counters, bool) { type role int const ( - roleSubscriber role = iota + 1 // receives, never publishes - rolePending // declared the principal as identity, no AUTH yet - rolePublisher // authenticated by its AUTH: may publish, receives nothing + roleSpectator role = iota + 1 // receives; sends nothing but, once a roster names it, its AUTH + rolePending // declared the principal — or a rostered address at a full fan-out set — no AUTH yet + rolePublisher // authenticated by its AUTH: publishes, and receives what others publish + roleRevoked // a publisher stream a roster dropped, or timed out pending: being closed, not read + roleEnded // a stream of a cohort its admin's EOS ended: being closed, not read ) type cohort struct { @@ -755,16 +1128,140 @@ type cohort struct { counters counters mtx sync.Mutex - cursor uint64 // the lowest DATA index accepted next + cursors map[string]uint64 // by publisher: the lowest DATA index accepted next + roster [][]byte // the latest accepted roster, the principal excluded + rosterCursor uint64 // the lowest ROSTER index accepted next + rosterFrame *pb.Broadcast // the latest accepted ROSTER frame: the first delivery inactiveSince time.Time members map[*member]struct{} - subscribers int + spectators int // the fan-out set + authSeq uint64 // orders publisher streams by their auth + ended bool // the admin's EOS was accepted } -// shutdown returns the cohort's streams, for the caller to reset. -func (co *cohort) shutdown() []*member { - co.mtx.Lock() - defer co.mtx.Unlock() +// rostered reports whether the latest roster names the address. Must be +// called with co.mtx held. +func (co *cohort) rostered(a []byte) bool { + return contains(co.roster, a) +} + +// receives reports whether deliveries are enqueued on a stream of the role. +func (r role) receives() bool { return r == roleSpectator || r == rolePublisher } + +// deliver enqueues an accepted frame on every receiving stream — for DATA, +// every one not bound to the sender's identity — and returns the streams whose +// queue was full. A ROSTER or EOS reaches the sender too: where it sits in the +// sender's stream is where it took effect, and every receiver, the admin +// included, admits by the rosters it receives. The send is +// non-blocking, so that one slow receiver cannot stall the cohort; a stream +// whose queue is full is returned and reset by the caller after co.mtx is +// released — reset ends the stream, whose handler then calls leave, which +// takes s.mtx and co.mtx, so resetting under the lock would deadlock. Must be +// called with co.mtx held. +func (co *cohort) deliver(f *pb.Broadcast, sender *member) []*member { + var overflow []*member + for m := range co.members { + if !m.role.receives() || f.Kind == pb.Kind_DATA && bytes.Equal(m.identity, sender.identity) { + continue + } + if !m.enqueue(f) { + overflow = append(overflow, m) + } + } + return overflow +} + +// upgrade makes a pending stream, or a spectator stream a roster names, a +// publisher stream, and returns the streams to reset: one whose queue was +// full, and the oldest publisher streams of the identity beyond the bound. +// Must be called with co.mtx held. +func (co *cohort) upgrade(m *member, maxStreams int) (resets, evicted []*member) { + if m.role == rolePending { + m.authTimer.Stop() + // a stream that starts receiving gets the latest roster first, unless + // it was given it as its cue + if co.rosterFrame != nil && m.given != co.rosterFrame && !m.enqueue(co.rosterFrame) { + resets = append(resets, m) + } + } else { + // a spectator leaves the fan-out set; it has the roster already + co.spectators-- + } + m.role = rolePublisher + co.authSeq++ + m.authSeq = co.authSeq + + var same []*member + for o := range co.members { + if o.role == rolePublisher && bytes.Equal(o.identity, m.identity) { + same = append(same, o) + } + } + for len(same) > maxStreams { + oldest := 0 + for i, o := range same { + if o.authSeq < same[oldest].authSeq { + oldest = i + } + } + evicted = append(evicted, same[oldest]) + same = append(same[:oldest], same[oldest+1:]...) + } + return resets, evicted +} + +// applyRoster makes an accepted roster the cohort's: it is delivered — to its +// sender too — and every publisher stream of an identity it drops is revoked: +// written the roster, then closed, what it sends meanwhile dropped; the +// cursors of dropped identities are forgotten. A spectator it names may send +// its AUTH. It returns the streams to reset for a full queue and the revoked +// streams to close. Must be called with co.mtx held. +func (co *cohort) applyRoster(f *pb.Broadcast, roster [][]byte, sender *member) (resets, closes []*member) { + principal := co.spec.Principal + co.roster = roster + co.rosterFrame = f + resets = co.deliver(f, sender) + for m := range co.members { + if bytes.Equal(m.identity, principal) { + continue + } + switch { + case m.role == roleSpectator && co.rostered(m.identity): + // delivered above: the cue for its AUTH + m.cued = true + case m.role == rolePending && co.rostered(m.identity): + // a rostered pending stream is given the latest roster + m.given = f + if !m.enqueue(f) { + resets = append(resets, m) + } + case m.role == rolePending: + // revoked before its auth: told, then closed + m.authTimer.Stop() + m.role = roleRevoked + if !m.enqueue(f) { + resets = append(resets, m) + continue + } + closes = append(closes, m) + case m.role == rolePublisher && !co.rostered(m.identity): + // revoked: it was delivered this roster above; closed once that + // is written + m.role = roleRevoked + closes = append(closes, m) + } + } + for k := range co.cursors { + if a := []byte(k); !bytes.Equal(a, principal) && !co.rostered(a) { + delete(co.cursors, k) + } + } + return resets, closes +} + +// end returns the cohort's streams for an EOS to close. Must be called with +// co.mtx held. +func (co *cohort) end() []*member { ms := make([]*member, 0, len(co.members)) for m := range co.members { ms = append(ms, m) @@ -772,11 +1269,37 @@ func (co *cohort) shutdown() []*member { return ms } -// member is one stream attached to a cohort. role is guarded by cohort.mtx. +// shutdown returns the cohort's streams, for the caller to reset. +func (co *cohort) shutdown() []*member { + co.mtx.Lock() + defer co.mtx.Unlock() + return co.end() +} + +// member is one stream attached to a cohort. role and authSeq are guarded by +// cohort.mtx. type member struct { role role + identity []byte challenge []byte queue chan *pb.Broadcast + ended chan struct{} // closed by an EOS: write the queue, then close reset func() + close func() + end func() authTimer *time.Timer + authSeq uint64 + cued bool // a roster naming the identity was enqueued on this spectator stream + given *pb.Broadcast // the roster last enqueued on a rostered pending stream +} + +// enqueue adds a frame to the stream's outbound queue, and reports false if +// the queue is full. +func (m *member) enqueue(f *pb.Broadcast) bool { + select { + case m.queue <- f: + return true + default: + return false + } } diff --git a/pkg/bps/bps_fuzz_test.go b/pkg/bps/bps_fuzz_test.go index 6b29965ad29..87d813e75cf 100644 --- a/pkg/bps/bps_fuzz_test.go +++ b/pkg/bps/bps_fuzz_test.go @@ -5,6 +5,7 @@ package bps import ( + "bytes" "testing" "github.com/ethersphere/bee/v2/pkg/bps/pb" @@ -14,9 +15,11 @@ import ( "github.com/ethersphere/bee/v2/pkg/swarm" ) -// FuzzVerify fuzzes the peer-supplied Broadcast validation path used by BPS. -// It exercises malformed frames and valid signed chunks and asserts the verifier -// never panics on hostile input. +// FuzzVerify fuzzes the peer-supplied Broadcast validation paths used by BPS — +// verify at a known owner, and recoverOwner at a subscriber — over every kind. +// It exercises malformed frames and valid signed chunks, asserts neither +// panics on hostile input, and that a chunk valid at an owner recovers that +// owner. func FuzzVerify(f *testing.F) { priv, err := crypto.GenerateSecp256k1Key() if err != nil { @@ -62,18 +65,53 @@ func FuzzVerify(f *testing.F) { f.Fatal("auth seed not the empty auth layout") } + rosterID, err := ID(KindRoster, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + rosterPayload, err := RosterPayload([][]byte{principal.Bytes()}) + if err != nil { + f.Fatal(err) + } + rosterCAC, err := cac.New(rosterPayload) + if err != nil { + f.Fatal(err) + } + rosterChunk, err := soc.New(rosterID, rosterCAC).Sign(signer) + if err != nil { + f.Fatal(err) + } + + eosID, err := ID(KindEOS, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + eosChunk, err := soc.New(eosID, authPayload).Sign(signer) + if err != nil { + f.Fatal(err) + } + f.Add(validChunk.Data(), uint8(pb.Kind_DATA), challenge, uint64(0), topic, principal.Bytes()) + f.Add(rosterChunk.Data(), uint8(pb.Kind_ROSTER), challenge, uint64(0), topic, principal.Bytes()) + f.Add(eosChunk.Data(), uint8(pb.Kind_EOS), challenge, uint64(0), topic, principal.Bytes()) f.Add(authChunk.Data(), uint8(pb.Kind_AUTH), challenge, uint64(0), topic, principal.Bytes()) f.Add([]byte{}, uint8(pb.Kind_DATA), make([]byte, ChallengeSize), uint64(0), topic, principal.Bytes()) f.Add(make([]byte, swarm.HashSize), uint8(pb.Kind_DATA), []byte{}, uint64(0), topic[:31], principal.Bytes()[:19]) f.Fuzz(func(t *testing.T, socData []byte, kind uint8, challenge []byte, index uint64, topic []byte, principal []byte) { - if kind > uint8(pb.Kind_AUTH) { + if kind > uint8(pb.Kind_ROSTER) { kind = uint8(pb.Kind_KIND_UNSPECIFIED) } - if err := verify(&pb.Broadcast{Soc: socData, Kind: pb.Kind(kind), Challenge: challenge, Index: index}, topic, principal); err != nil { + frame := &pb.Broadcast{Soc: socData, Kind: pb.Kind(kind), Challenge: challenge, Index: index} + owner, recErr := recoverOwner(frame, topic) + _, _ = rosterOf(socData) + if err := verify(frame, topic, principal); err != nil { return } + // a chunk valid at an owner recovers that owner + if recErr != nil || !bytes.Equal(owner, principal) { + t.Fatalf("valid at %x, recovered %x (%v)", principal, owner, recErr) + } if len(topic) != swarm.HashSize || len(principal) != crypto.AddressSize || len(challenge) != ChallengeSize { return @@ -85,3 +123,45 @@ func FuzzVerify(f *testing.F) { } }) } + +// FuzzRoster fuzzes the decoding of a ROSTER chunk's payload, which a peer +// supplies: it never panics, every address it returns is 20 bytes, and what +// it returns encodes and decodes to itself. +func FuzzRoster(f *testing.F) { + off := swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + header := make([]byte, off) + one, err := RosterPayload([][]byte{make([]byte, crypto.AddressSize)}) + if err != nil { + f.Fatal(err) + } + f.Add(append(append([]byte{}, header...), one...)) + f.Add(header) + f.Add([]byte{}) + f.Add(append(append([]byte{}, header...), 0x0a, 0x02, 0x01, 0x02)) + f.Add(append(append([]byte{}, header...), 0x0a, 0xff, 0xff, 0xff, 0xff, 0x0f)) + + f.Fuzz(func(t *testing.T, chunk []byte) { + roster, err := ParseRoster(chunk) + if err != nil { + return + } + for _, a := range roster { + if len(a) != crypto.AddressSize { + t.Fatalf("address of %d bytes", len(a)) + } + } + payload, err := RosterPayload(roster) + if err != nil { + t.Fatal(err) + } + again, err := ParseRoster(append(append([]byte{}, chunk[:off]...), payload...)) + if err != nil || len(again) != len(roster) { + t.Fatalf("round trip: %d of %d (%v)", len(again), len(roster), err) + } + for i := range roster { + if !bytes.Equal(again[i], roster[i]) { + t.Fatalf("round trip: entry %d differs", i) + } + } + }) +} diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index 0e1785e2a6c..7f91183346e 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -28,16 +28,21 @@ import ( const waitFor = 5 * time.Second type blocklister struct { - mu sync.Mutex - peers map[string]string + mu sync.Mutex + peers map[string]string + durations map[string]time.Duration } func (b *blocklister) NetworkStatus() p2p.NetworkStatus { return p2p.NetworkStatusAvailable } -func (b *blocklister) Blocklist(overlay swarm.Address, _ time.Duration, reason string) error { +func (b *blocklister) Blocklist(overlay swarm.Address, d time.Duration, reason string) error { b.mu.Lock() defer b.mu.Unlock() b.peers[overlay.ByteString()] = reason + if b.durations == nil { + b.durations = make(map[string]time.Duration) + } + b.durations[overlay.ByteString()] = d return nil } @@ -486,12 +491,21 @@ func TestInvalidDelivery(t *testing.T) { }}, } rec := streamtest.New(streamtest.WithProtocols(broker), streamtest.WithBaseAddr(swarm.RandAddress(t))) - c := bps.New(rec, nil, false, log.Noop, bps.Options{}) + bl := &blocklister{peers: make(map[string]string)} + c := bps.New(rec, bl, false, log.Noop, bps.Options{}) sub := e.join(c, swarm.RandAddress(t).Bytes()[:20]) expectDone(t, sub) if err := sub.Err(); !errors.Is(err, bps.ErrInvalidDelivery) { t.Fatalf("got %v, want %v", err, bps.ErrInvalidDelivery) } + // the broker is blocklisted for the longest period there is: forever + eventually(t, func() bool { return bl.blocklisted(e.brokerAddr) }) + bl.mu.Lock() + d := bl.durations[e.brokerAddr.ByteString()] + bl.mu.Unlock() + if d != 0 { + t.Fatalf("broker blocklisted for %v, want forever (0)", d) + } } func TestJoinRejected(t *testing.T) { diff --git a/pkg/bps/export_test.go b/pkg/bps/export_test.go index 58f68d2fa7e..148efd2f87c 100644 --- a/pkg/bps/export_test.go +++ b/pkg/bps/export_test.go @@ -15,3 +15,7 @@ var ( ErrNotPublisher = errNotPublisher ErrInvalidDelivery = errInvalidDelivery ) + +var ( + ErrChannelEnded = errChannelEnded +) diff --git a/pkg/bps/flush_test.go b/pkg/bps/flush_test.go new file mode 100644 index 00000000000..1099c8474bb --- /dev/null +++ b/pkg/bps/flush_test.go @@ -0,0 +1,419 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/coreos/go-semver/semver" + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +var errFlushStreamReset = errors.New("flush stream reset") + +// flushStream is a broker-side stream whose peer is driven by the test: the +// peer's frames are written into a pipe the broker reads, and the broker's +// writes land in a buffer — or, once stalled, block as on a transport whose +// peer has stopped reading, until the stream is reset or closed. +type flushStream struct { + r *io.PipeReader + pw *io.PipeWriter + + mu sync.Mutex + out bytes.Buffer + stall bool + stalled chan struct{} // closed when a write blocks + unblock chan struct{} // closed by Reset or Close + + reset, closed chan struct{} + stallOnce, unblockOnce sync.Once + resetOnce, closedOnce sync.Once +} + +func newFlushStream() *flushStream { + r, pw := io.Pipe() + return &flushStream{ + r: r, pw: pw, + stalled: make(chan struct{}), + unblock: make(chan struct{}), + reset: make(chan struct{}), + closed: make(chan struct{}), + } +} + +func (s *flushStream) Read(p []byte) (int, error) { return s.r.Read(p) } + +func (s *flushStream) Write(p []byte) (int, error) { + s.mu.Lock() + if !s.stall { + defer s.mu.Unlock() + return s.out.Write(p) + } + s.mu.Unlock() + s.stallOnce.Do(func() { close(s.stalled) }) + <-s.unblock + return 0, errFlushStreamReset +} + +func (s *flushStream) end() { + s.unblockOnce.Do(func() { close(s.unblock) }) + _ = s.r.Close() +} + +func (s *flushStream) Reset() error { + s.resetOnce.Do(func() { close(s.reset) }) + s.end() + return nil +} + +func (s *flushStream) Close() error { + s.closedOnce.Do(func() { close(s.closed) }) + s.end() + return nil +} + +func (s *flushStream) FullClose() error { return s.Close() } +func (s *flushStream) Headers() p2p.Headers { return nil } +func (s *flushStream) ResponseHeaders() p2p.Headers { return nil } +func (s *flushStream) Version() (*semver.Version, error) { return nil, nil } + +// stallWrites makes every later broker write block. +func (s *flushStream) stallWrites() { + s.mu.Lock() + s.stall = true + s.mu.Unlock() +} + +// frames returns what the broker has written so far, after the Ack. +func (s *flushStream) frames(t *testing.T) []*pb.Broadcast { + t.Helper() + s.mu.Lock() + r := protobuf.NewReader(bytes.NewReader(append([]byte{}, s.out.Bytes()...))) + s.mu.Unlock() + var ack pb.Ack + if err := r.ReadMsg(&ack); err != nil { + t.Fatal(err) + } + var fs []*pb.Broadcast + for { + var f pb.Broadcast + if err := r.ReadMsg(&f); err != nil { + return fs + } + fs = append(fs, &f) + } +} + +// flushEnv runs the broker's handler over flushStreams. +type flushEnv struct { + t *testing.T + svc *Service + admin crypto.Signer + spec *pb.CohortSpec +} + +func newFlushEnv(t *testing.T, o Options) *flushEnv { + t.Helper() + svc := New(nil, nil, true, log.Noop, o) + t.Cleanup(func() { _ = svc.Close() }) + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + principal, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + topic := make([]byte, swarm.HashSize) + copy(topic, "bps-flush-topic") + return &flushEnv{t: t, svc: svc, admin: signer, + spec: &pb.CohortSpec{Topic: topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: principal.Bytes()}} +} + +func newSigner(t *testing.T) (crypto.Signer, []byte) { + t.Helper() + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + addr, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + return signer, addr.Bytes() +} + +// peer is the test's end of one stream. +type peer struct { + e *flushEnv + st *flushStream + w protobuf.Writer + signer crypto.Signer + challenge []byte + done chan struct{} // the handler returned +} + +// join attaches a stream declaring identity and returns once the Ack is written. +func (e *flushEnv) join(signer crypto.Signer, identity []byte) *peer { + e.t.Helper() + st := newFlushStream() + p := &peer{e: e, st: st, w: protobuf.NewWriter(st.pw), signer: signer, done: make(chan struct{})} + go func() { + defer close(p.done) + _ = e.svc.handler(context.Background(), p2p.Peer{Address: swarm.RandAddress(e.t)}, st) + }() + go func() { _ = p.w.WriteMsg(&pb.Join{Cohort: e.spec, Identity: identity}) }() + deadline := time.Now().Add(waitFlush) + for { + st.mu.Lock() + var ack pb.Ack + err := protobuf.NewReader(bytes.NewReader(st.out.Bytes())).ReadMsg(&ack) + st.mu.Unlock() + if err == nil { + if ack.Status != pb.Status_OK { + e.t.Fatalf("join: %s", ack.Status) + } + p.challenge = ack.Challenge + return p + } + if time.Now().After(deadline) { + e.t.Fatal("no ack") + } + time.Sleep(5 * time.Millisecond) + } +} + +// send signs a chunk of kind at index under the stream's challenge and writes it. +func (p *peer) send(kind pb.Kind, index uint64, payload []byte) error { + chunkID, err := id(kind, p.e.spec.Topic, p.challenge, index) + if err != nil { + return err + } + ch, err := cac.New(payload) + if err != nil { + return err + } + c, err := soc.New(chunkID, ch).Sign(p.signer) + if err != nil { + return err + } + return p.w.WriteMsg(&pb.Broadcast{Soc: c.Data(), Kind: kind, Challenge: p.challenge, Index: index}) +} + +func (p *peer) mustSend(kind pb.Kind, index uint64, payload []byte) { + p.e.t.Helper() + if err := p.send(kind, index, payload); err != nil { + p.e.t.Fatal(err) + } +} + +func (e *flushEnv) roster(admin *peer, index uint64, addrs ...[]byte) { + e.t.Helper() + payload, err := RosterPayload(addrs) + if err != nil { + e.t.Fatal(err) + } + admin.mustSend(pb.Kind_ROSTER, index, payload) +} + +func (e *flushEnv) counters() Counters { + e.t.Helper() + key, err := e.spec.Marshal() + if err != nil { + e.t.Fatal(err) + } + e.svc.mtx.Lock() + co := e.svc.cohorts[string(key)] + e.svc.mtx.Unlock() + if co == nil { + e.t.Fatal("no live cohort") + } + return co.counters.snapshot() +} + +const waitFlush = 5 * time.Second + +func waitClosed(t *testing.T, c <-chan struct{}, what string) { + t.Helper() + select { + case <-c: + case <-time.After(waitFlush): + t.Fatalf("%s: timed out", what) + } +} + +func notClosed(t *testing.T, c <-chan struct{}, what string) { + t.Helper() + select { + case <-c: + t.Fatalf("%s: unexpected", what) + default: + } +} + +func eventuallyTrue(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(waitFlush) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not met") + } + time.Sleep(5 * time.Millisecond) + } +} + +// TestFlushDeadline covers the flush deadline of a stream being closed — by +// an EOS or by a revocation — on its write side, where the peer has stopped +// reading and the broker's write blocks, and on its read side, where the peer +// goes on sending or never closes. +func TestFlushDeadline(t *testing.T) { + t.Parallel() + const flush = 300 * time.Millisecond + + t.Run("write: an EOS resets a stream whose peer stopped reading, at the deadline", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + sub.st.stallWrites() + + // the update blocks the subscriber's writer mid-write, outside any flush + admin.mustSend(pb.Kind_DATA, 0, []byte("stuck")) + waitClosed(t, sub.st.stalled, "the write did not block") + + start := time.Now() + admin.mustSend(pb.Kind_EOS, 0, nil) + waitClosed(t, sub.st.reset, "the stalled stream was not reset") + if elapsed := time.Since(start); elapsed < flush*3/4 { + t.Fatalf("reset after %v, before the flush deadline %v", elapsed, flush) + } + waitClosed(t, sub.done, "the stalled stream's handler did not return") + notClosed(t, sub.st.closed, "the stalled stream closed") + + // the sender, which reads, takes its EOS and is closed, not reset + waitClosed(t, admin.st.closed, "the admin's stream was not closed") + notClosed(t, admin.st.reset, "the admin's stream reset") + waitClosed(t, admin.done, "the admin's handler did not return") + fs := admin.st.frames(t) + if len(fs) == 0 || fs[len(fs)-1].Kind != pb.Kind_EOS { + t.Fatalf("the admin's stream did not end on its EOS: %v", fs) + } + }) + + t.Run("write: a revocation resets a publisher stream whose peer stopped reading, at the deadline", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + seatSigner, seatAddr := newSigner(t) + e.roster(admin, 0, seatAddr) + seat := e.join(seatSigner, seatAddr) + eventuallyTrue(t, func() bool { return len(seat.st.frames(t)) > 0 }) // the cue + seat.mustSend(pb.Kind_AUTH, 0, nil) + eventuallyTrue(t, func() bool { + fs := admin.st.frames(t) + return len(fs) > 0 && fs[len(fs)-1].Kind == pb.Kind_ROSTER + }) + seat.st.stallWrites() + + admin.mustSend(pb.Kind_DATA, 0, []byte("stuck")) + waitClosed(t, seat.st.stalled, "the write did not block") + + start := time.Now() + e.roster(admin, 1) + waitClosed(t, seat.st.reset, "the stalled revoked stream was not reset") + if elapsed := time.Since(start); elapsed < flush*3/4 { + t.Fatalf("reset after %v, before the flush deadline %v", elapsed, flush) + } + waitClosed(t, seat.done, "the revoked stream's handler did not return") + notClosed(t, admin.st.reset, "the admin's stream reset") + }) + + t.Run("read: a revoked peer that keeps sending is closed, not reset, its frames dropped", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: 10 * flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + seatSigner, seatAddr := newSigner(t) + e.roster(admin, 0, seatAddr) + seat := e.join(seatSigner, seatAddr) + eventuallyTrue(t, func() bool { return len(seat.st.frames(t)) > 0 }) + seat.mustSend(pb.Kind_AUTH, 0, nil) + eventuallyTrue(t, func() bool { + fs := admin.st.frames(t) + return len(fs) > 0 && fs[len(fs)-1].Kind == pb.Kind_ROSTER + }) + + // the seat floods its stream with updates until it is closed + flooding := make(chan struct{}) + go func() { + defer close(flooding) + for i := uint64(0); ; i++ { + if seat.send(pb.Kind_DATA, i, []byte("flood")) != nil { + return + } + } + }() + eventuallyTrue(t, func() bool { return e.counters().Retransmit == 0 && len(admin.st.frames(t)) > 2 }) + + start := time.Now() + e.roster(admin, 1) + waitClosed(t, seat.st.closed, "the revoked stream was not closed") + if elapsed := time.Since(start); elapsed >= 10*flush { + t.Fatalf("closed after %v: the flood held it to the deadline", elapsed) + } + notClosed(t, seat.st.reset, "the revoked stream reset") + waitClosed(t, seat.done, "the revoked stream's handler did not return") + waitClosed(t, flooding, "the flood did not stop") + + c := e.counters() + if c.WrongStream != 0 || c.InvalidSOC != 0 { + t.Fatalf("a revoked peer's frames punished: %+v", c) + } + fs := seat.st.frames(t) + if last := fs[len(fs)-1]; last.Kind != pb.Kind_ROSTER || last.Index != 1 { + t.Fatalf("the revoked stream did not end on the reduced roster: %s %d", last.Kind, last.Index) + } + }) + + t.Run("read: a silent peer that never closes its side does not hold an ended stream", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: 10 * flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + + start := time.Now() + admin.mustSend(pb.Kind_EOS, 0, nil) + waitClosed(t, sub.st.closed, "the ended stream was not closed") + waitClosed(t, sub.done, "the ended stream's handler did not return") + if elapsed := time.Since(start); elapsed >= 10*flush { + t.Fatalf("closed after %v: the silent peer held it to the deadline", elapsed) + } + notClosed(t, sub.st.reset, "the ended stream reset") + fs := sub.st.frames(t) + if len(fs) != 1 || fs[0].Kind != pb.Kind_EOS { + t.Fatalf("the ended stream got %v, want the EOS alone", fs) + } + }) +} diff --git a/pkg/bps/id.go b/pkg/bps/id.go index 54801b202b7..619b93903e3 100644 --- a/pkg/bps/id.go +++ b/pkg/bps/id.go @@ -24,9 +24,13 @@ const ChallengeSize = 32 var servicePrefix = []byte("bps-service:v1") // errInvalidSOC is returned for a chunk that does not have the id derived -// from its frame, or does not validate as the principal's single-owner chunk. +// from its frame, or does not validate as its owner's single-owner chunk. var errInvalidSOC = errors.New("bps: invalid soc") +// errInvalidRoster is returned for a ROSTER chunk whose payload does not +// decode as a Roster of 20-byte addresses. +var errInvalidRoster = errors.New("bps: invalid roster") + // Kind is what a chunk on the wire is. type Kind int @@ -34,8 +38,14 @@ const ( KindUnspecified Kind = iota KindData // an update of the stream's session feed // KindAuth is the empty chunk at index 0 of the session's AUTH feed: it authenticates - // a pending stream, is a heartbeat on a publisher stream, and is never delivered. + // a pending stream, or upgrades in place a spectator stream on which a roster naming + // its identity was delivered; it is a heartbeat on a publisher stream, and is never + // delivered. KindAuth + // KindEOS is the empty chunk at index 0 of the admin's EOS feed: the channel is ended. + KindEOS + // KindRoster is the admin's publisher set as of its index; its payload is a Roster. + KindRoster ) func (k Kind) proto() pb.Kind { @@ -44,6 +54,10 @@ func (k Kind) proto() pb.Kind { return pb.Kind_DATA case KindAuth: return pb.Kind_AUTH + case KindEOS: + return pb.Kind_EOS + case KindRoster: + return pb.Kind_ROSTER default: return pb.Kind_KIND_UNSPECIFIED } @@ -79,44 +93,114 @@ func id(kind pb.Kind, topic, challenge []byte, index uint64) ([]byte, error) { } // Verify checks that the frame fields kind, challenge and index give the id of -// the chunk, and that the chunk validates as the principal's single-owner chunk -// at keccak256(id | principal). An AUTH chunk must moreover be at index 0 and empty. -func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, principal []byte) error { - return verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, principal) +// the chunk, and that the chunk validates as owner's single-owner chunk at +// keccak256(id | owner). An AUTH or EOS chunk must moreover be at index 0 and empty. +func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, owner []byte) error { + return verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, owner) +} + +// service reports whether the kind is one of the admin's service kinds, whose +// owner is forced to the principal. +func service(kind pb.Kind) bool { + return kind == pb.Kind_EOS || kind == pb.Kind_ROSTER +} + +// empty reports whether the kind is an empty chunk at index 0 of its feed. +func empty(kind pb.Kind) bool { + return kind == pb.Kind_AUTH || kind == pb.Kind_EOS } -func verify(f *pb.Broadcast, topic, principal []byte) error { +// checkFrame runs the checks that need no owner: the challenge length, the +// index, and the id the frame fields derive. +func checkFrame(f *pb.Broadcast, topic []byte) ([]byte, error) { if len(f.Challenge) != ChallengeSize { - return fmt.Errorf("challenge length %d: %w", len(f.Challenge), errInvalidSOC) + return nil, fmt.Errorf("challenge length %d: %w", len(f.Challenge), errInvalidSOC) } // the last index is one no feed has: the cursor, set to index+1, never wraps if f.Index == math.MaxUint64 { - return fmt.Errorf("index %d: %w", f.Index, errInvalidSOC) + return nil, fmt.Errorf("index %d: %w", f.Index, errInvalidSOC) } - // a session has one AUTH chunk, at index 0 of its AUTH feed - if f.Kind == pb.Kind_AUTH && f.Index != 0 { - return fmt.Errorf("auth chunk at index %d: %w", f.Index, errInvalidSOC) + // a session has one AUTH chunk, and a channel one EOS, at index 0 of its feed + if empty(f.Kind) && f.Index != 0 { + return nil, fmt.Errorf("%s chunk at index %d: %w", f.Kind, f.Index, errInvalidSOC) } want, err := id(f.Kind, topic, f.Challenge, f.Index) if err != nil { - return err + return nil, err } if len(f.Soc) < swarm.HashSize || !bytes.Equal(f.Soc[:swarm.HashSize], want) { - return fmt.Errorf("id not derived from frame: %w", errInvalidSOC) + return nil, fmt.Errorf("id not derived from frame: %w", errInvalidSOC) } - addr, err := soc.CreateAddress(want, principal) + if empty(f.Kind) { + // span 0, no payload + if len(f.Soc) != swarm.HashSize+swarm.SocSignatureSize+swarm.SpanSize || + binary.LittleEndian.Uint64(f.Soc[swarm.HashSize+swarm.SocSignatureSize:]) != 0 { + return nil, fmt.Errorf("%s chunk not empty: %w", f.Kind, errInvalidSOC) + } + } + return want, nil +} + +func verify(f *pb.Broadcast, topic, owner []byte) error { + want, err := checkFrame(f, topic) + if err != nil { + return err + } + addr, err := soc.CreateAddress(want, owner) if err != nil { return err } if !soc.Valid(swarm.NewChunk(addr, f.Soc)) { - return fmt.Errorf("not the principal's soc: %w", errInvalidSOC) + return fmt.Errorf("not the owner's soc: %w", errInvalidSOC) } - if f.Kind == pb.Kind_AUTH { - // span 0, no payload - if len(f.Soc) != swarm.HashSize+swarm.SocSignatureSize+swarm.SpanSize || - binary.LittleEndian.Uint64(f.Soc[swarm.HashSize+swarm.SocSignatureSize:]) != 0 { - return fmt.Errorf("auth chunk not empty: %w", errInvalidSOC) + return nil +} + +// recoverOwner checks the frame as verify does and returns the owner the +// chunk's signature recovers: the chunk is then valid at keccak256(id | owner). +func recoverOwner(f *pb.Broadcast, topic []byte) ([]byte, error) { + if _, err := checkFrame(f, topic); err != nil { + return nil, err + } + s, err := soc.FromChunk(swarm.NewChunk(swarm.ZeroAddress, f.Soc)) + if err != nil { + return nil, fmt.Errorf("%w: %w", errInvalidSOC, err) + } + return s.OwnerAddress(), nil +} + +// decodeRoster returns the publisher set a validated ROSTER chunk carries. +func decodeRoster(chunk []byte) ([][]byte, error) { + off := swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + if len(chunk) < off { + return nil, errInvalidRoster + } + var r pb.Roster + if err := r.Unmarshal(chunk[off:]); err != nil { + return nil, fmt.Errorf("%w: %w", errInvalidRoster, err) + } + for _, a := range r.Publishers { + if len(a) != crypto.AddressSize { + return nil, fmt.Errorf("address length %d: %w", len(a), errInvalidRoster) } } - return nil + return r.Publishers, nil +} + +// rosterOf returns the publisher set of a validated ROSTER chunk, and false if +// its payload does not decode: the admin's mistake, which is dropped, not an error. +func rosterOf(chunk []byte) ([][]byte, bool) { + roster, err := decodeRoster(chunk) + return roster, err == nil +} + +// ParseRoster returns the publisher set a ROSTER chunk's payload names, or an +// error if it does not decode as a Roster of 20-byte addresses. +func ParseRoster(chunk []byte) ([][]byte, error) { + return decodeRoster(chunk) +} + +// RosterPayload returns the payload of a ROSTER chunk naming the publishers. +func RosterPayload(publishers [][]byte) ([]byte, error) { + return (&pb.Roster{Publishers: publishers}).Marshal() } diff --git a/pkg/bps/metrics.go b/pkg/bps/metrics.go index 0473653e6dd..2e3fafcf9e8 100644 --- a/pkg/bps/metrics.go +++ b/pkg/bps/metrics.go @@ -14,13 +14,16 @@ import ( // Counters are the per-cohort counters of the silent outcomes SWIP-74 makes a // broker expose. type Counters struct { - WrongStream uint64 // a frame from a subscriber stream - UnknownKind uint64 // a frame of a kind BPS-lite does not define + WrongStream uint64 // a frame its stream may not send + UnknownKind uint64 // a frame of a kind the broker does not define WrongChallenge uint64 // a frame whose challenge is not its stream's InvalidSOC uint64 // a chunk that does not validate at the derived id AuthTimeout uint64 // a pending stream disconnected at the auth timeout Retransmit uint64 // a DATA frame below the cursor - QueueReset uint64 // a subscriber stream reset for a full queue + QueueReset uint64 // a receiving stream reset for a full queue + InvalidRoster uint64 // a valid ROSTER chunk whose payload does not decode + Revoked uint64 // a frame from a revoked stream, dropped unread + Ended uint64 // a frame from a stream of a cohort an EOS ended, dropped unread } type counters struct { @@ -31,6 +34,9 @@ type counters struct { authTimeout atomic.Uint64 retransmit atomic.Uint64 queueReset atomic.Uint64 + invalidRoster atomic.Uint64 + revoked atomic.Uint64 + ended atomic.Uint64 } func (c *counters) snapshot() Counters { @@ -42,6 +48,9 @@ func (c *counters) snapshot() Counters { AuthTimeout: c.authTimeout.Load(), Retransmit: c.retransmit.Load(), QueueReset: c.queueReset.Load(), + InvalidRoster: c.invalidRoster.Load(), + Revoked: c.revoked.Load(), + Ended: c.ended.Load(), } } @@ -65,7 +74,7 @@ func newMetrics() metrics { Namespace: m.Namespace, Subsystem: subsystem, Name: "accepted_updates", - Help: "DATA updates accepted by the broker.", + Help: "DATA updates accepted by the broker, from every publisher.", }), Cohorts: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: m.Namespace, diff --git a/pkg/bps/multipublisher_test.go b/pkg/bps/multipublisher_test.go new file mode 100644 index 00000000000..bfe053d9922 --- /dev/null +++ b/pkg/bps/multipublisher_test.go @@ -0,0 +1,538 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps_test + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// seat is a key the admin may roster. +type seat struct { + signer crypto.Signer + addr []byte +} + +func (e *env) newSeat() seat { + e.t.Helper() + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + e.t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + addr, err := signer.EthereumAddress() + if err != nil { + e.t.Fatal(err) + } + return seat{signer: signer, addr: addr.Bytes()} +} + +// joinAs joins as the seat's address from a fresh overlay. +func (e *env) joinAs(st seat) (bps.Session, swarm.Address) { + e.t.Helper() + overlay := swarm.RandAddress(e.t) + c, _ := e.client(overlay) + return e.join(c, st.addr), overlay +} + +// grant publishes a roster and waits until the broker has accepted it. +func (e *env) grant(admin bps.Session, index uint64, seats ...seat) { + e.t.Helper() + probe := e.subscribe() + e.roster(admin, index, seats...) + for { + m := <-probe.Messages() + if m.Kind == bps.KindRoster && m.Index == index { + return + } + } +} + +// publishAs signs a chunk with the seat's key under the session's challenge +// and publishes it. +func (e *env) publishAs(st seat, s bps.Session, kind bps.Kind, index uint64, payload []byte) []byte { + e.t.Helper() + c := e.chunk(st.signer, kind, s.Challenge(), index, payload) + if err := s.Publish(context.Background(), kind, index, c); err != nil { + e.t.Fatal(err) + } + return c +} + +// roster publishes the admin's roster at index on the admin's publisher stream. +func (e *env) roster(admin bps.Session, index uint64, seats ...seat) []byte { + e.t.Helper() + addrs := make([][]byte, 0, len(seats)) + for _, st := range seats { + addrs = append(addrs, st.addr) + } + payload, err := bps.RosterPayload(addrs) + if err != nil { + e.t.Fatal(err) + } + return e.publish(admin, bps.KindRoster, index, payload) +} + +func waitGranted(t *testing.T, s bps.Session) { + t.Helper() + select { + case <-s.Granted(): + case <-time.After(waitFor): + t.Fatal("not granted") + } +} + +func expectKind(t *testing.T, s bps.Session, kind bps.Kind, owner []byte, index uint64) bps.Message { + t.Helper() + select { + case m, ok := <-s.Messages(): + if !ok { + t.Fatalf("stream ended: %v", s.Err()) + } + if m.Kind != kind || !bytes.Equal(m.Owner, owner) || m.Index != index { + t.Fatalf("got kind %d owner %x index %d, want kind %d owner %x index %d", m.Kind, m.Owner, m.Index, kind, owner, index) + } + return m + case <-time.After(waitFor): + t.Fatalf("timed out waiting for kind %d index %d", kind, index) + } + return bps.Message{} +} + +// drain discards a session's deliveries, so that its stream keeps reading. +func drain(s bps.Session) { + go func() { + for range s.Messages() { + } + }() +} + +func TestGrant(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + sub := e.subscribe() + admin, _ := e.publisher() + + e.roster(admin, 0, st) + m := expectKind(t, sub, bps.KindRoster, e.principal, 0) + if len(m.Roster) != 1 || !bytes.Equal(m.Roster[0], st.addr) { + t.Fatalf("roster %x", m.Roster) + } + // the roster is delivered back to the admin: it admits by it from there + expectKind(t, admin, bps.KindRoster, e.principal, 0) + + // the seat joins as a spectator; the latest roster, naming it, is its cue + seatSess, _ := e.joinAs(st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + c0 := e.publishAs(st, seatSess, bps.KindData, 0, []byte("from the seat")) + + got := expectKind(t, sub, bps.KindData, st.addr, 0) + if !bytes.Equal(got.SOC, c0) { + t.Fatal("wrong chunk") + } + // the admin's publisher stream receives the seat's update; the seat not its own + expectKind(t, admin, bps.KindData, st.addr, 0) + a0 := e.publish(admin, bps.KindData, 0, []byte("from the admin")) + got = expectKind(t, seatSess, bps.KindData, e.principal, 0) + if !bytes.Equal(got.SOC, a0) { + t.Fatal("wrong chunk") + } + expectKind(t, sub, bps.KindData, e.principal, 0) + expectNoMessage(t, admin) +} + +// TestSeatBeforeRoster is the startup order the review found: the seat joins +// before the broker holds any roster, and is upgraded in place by its AUTH +// once the first roster names it. +func TestSeatBeforeRoster(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + sub := e.subscribe() + + seatSess, overlay := e.joinAs(st) + admin, _ := e.publisher() + drain(admin) + e.roster(admin, 0, st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + c := e.publishAs(st, seatSess, bps.KindData, 5, []byte("upgraded in place")) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + got := expectKind(t, sub, bps.KindData, st.addr, 5) + if !bytes.Equal(got.SOC, c) { + t.Fatal("wrong chunk") + } + if e.bl.blocklisted(overlay) || e.counters().WrongStream != 0 { + t.Fatal("an upgrade in place is not a violation") + } +} + +// TestRosteredListens: a rostered address that never authenticates is a +// spectator like any other, and is not disconnected at the auth deadline. +func TestRosteredListens(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{AuthWaitTimeout: 100 * time.Millisecond}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + listener, overlay := e.joinAs(st) + expectKind(t, listener, bps.KindRoster, e.principal, 0) + time.Sleep(300 * time.Millisecond) + e.publish(admin, bps.KindData, 0, []byte("heard")) + expectKind(t, listener, bps.KindData, e.principal, 0) + if e.bl.blocklisted(overlay) || e.counters().AuthTimeout != 0 { + t.Fatal("a listening rostered address is not timed out") + } +} + +func TestUnrosteredAuth(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + _, _ = e.publisher() + overlay := swarm.RandAddress(t) + w, _, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Identity: st.addr}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + auth := e.chunk(st.signer, bps.KindAuth, ack.Challenge, 0, nil) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}); err != nil { + t.Fatal(err) + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) && e.counters().WrongStream == 1 }) +} + +// TestRevokedSpectatorAuth: a spectator sends its AUTH after the roster naming +// it, but a roster revoking it reaches the broker first: the stream is closed, +// nothing is punished. +func TestRevokedSpectatorAuth(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Identity: st.addr}) + var cue pb.Broadcast + if err := r.ReadMsg(&cue); err != nil || cue.Kind != pb.Kind_ROSTER { + t.Fatalf("want the roster as the cue, got %s %v", cue.Kind, err) + } + e.grant(admin, 1) + auth := e.chunk(st.signer, bps.KindAuth, ack.Challenge, 0, nil) + _ = w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}) + eventually(t, func() bool { return e.counters().Revoked == 1 }) + if e.bl.blocklisted(overlay) || e.counters().WrongStream != 0 { + t.Fatal("an AUTH racing its revocation is not punished") + } +} + +func TestRevocation(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + sub := e.subscribe() + e.roster(admin, 0, st) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + + // the seat authenticates on a raw stream, to send after its revocation + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Identity: st.addr}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + var cue pb.Broadcast + if err := r.ReadMsg(&cue); err != nil || cue.Kind != pb.Kind_ROSTER { + t.Fatalf("want the roster as the cue, got %s %v", cue.Kind, err) + } + send := func(kind bps.Kind, index uint64, payload []byte) { + t.Helper() + c := e.chunk(st.signer, kind, ack.Challenge, index, payload) + f := &pb.Broadcast{Kind: pb.Kind(kind), Challenge: ack.Challenge, Index: index, Soc: c} + _ = w.WriteMsg(f) + } + send(bps.KindAuth, 0, nil) + send(bps.KindData, 0, []byte("before")) + expectKind(t, sub, bps.KindData, st.addr, 0) + + // the reduced roster revokes the seat: the stream is written that roster + // and closed; what it sends meanwhile is dropped, unpunished + e.grant(admin, 1) + send(bps.KindData, 1, []byte("in flight")) + m := expectKind(t, sub, bps.KindRoster, e.principal, 1) + if len(m.Roster) != 0 { + t.Fatalf("roster %x", m.Roster) + } + var next pb.Broadcast + if err := r.ReadMsg(&next); err != nil || next.Kind != pb.Kind_ROSTER || next.Index != 1 { + t.Fatalf("want the reduced roster, got %s %d %v", next.Kind, next.Index, err) + } + if err := r.ReadMsg(&next); err == nil { + t.Fatalf("want the stream closed, got %s", next.Kind) + } + expectNoMessage(t, sub) + if e.bl.blocklisted(overlay) || e.counters().WrongStream != 0 { + t.Fatal("a revoked peer is not punished") + } + if pos := sub.Position(); len(pos.Roster) != 0 || pos.RosterCursor != 2 { + t.Fatalf("position %+v", pos) + } + + // rejoining, the seat is a spectator that may not authenticate + spectator, _ := e.joinAs(st) + expectKind(t, spectator, bps.KindRoster, e.principal, 1) + select { + case <-spectator.Granted(): + t.Fatal("a revoked identity is not granted") + default: + } +} + +// TestRevokeInFlight: the admin revokes a seat whose update the broker accepted +// just before the roster; the admin's own session, admitting by the echoed +// roster, takes the update and is not reset. +func TestRevokeInFlight(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + e.roster(admin, 0, st) + expectKind(t, admin, bps.KindRoster, e.principal, 0) + seatSess, _ := e.joinAs(st) + drain(seatSess) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + e.publishAs(st, seatSess, bps.KindData, 0, []byte("last")) + time.Sleep(50 * time.Millisecond) // the broker accepts the update first + e.roster(admin, 1) + expectKind(t, admin, bps.KindData, st.addr, 0) + expectKind(t, admin, bps.KindRoster, e.principal, 1) + select { + case <-admin.Done(): + t.Fatalf("the admin's stream ended: %v", admin.Err()) + default: + } +} + +func TestRevokedSession(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + seatSess, _ := e.joinAs(st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + + e.roster(admin, 1) + expectKind(t, seatSess, bps.KindRoster, e.principal, 1) + expectDone(t, seatSess) + if err := seatSess.Publish(context.Background(), bps.KindData, 0, nil); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("got %v, want %v", err, bps.ErrNotPublisher) + } +} + +func TestCursorsPerPublisher(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + sub := e.subscribe() + e.roster(admin, 0, st) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + seatSess, _ := e.joinAs(st) + drain(seatSess) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + + // each publisher has its own cursor + e.publish(admin, bps.KindData, 3, []byte("a3")) + expectKind(t, sub, bps.KindData, e.principal, 3) + e.publishAs(st, seatSess, bps.KindData, 0, []byte("s0")) + expectKind(t, sub, bps.KindData, st.addr, 0) + // below its own cursor: a retransmit + e.publishAs(st, seatSess, bps.KindData, 0, []byte("s0")) + eventually(t, func() bool { return e.counters().Retransmit == 1 }) + expectNoMessage(t, sub) + + pos := sub.Position() + if pos.Cursors[string(e.principal)] != 4 || pos.Cursors[string(st.addr)] != 1 { + t.Fatalf("cursors %v", pos.Cursors) + } +} + +func TestRosterFeed(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + sub := e.subscribe() + + e.roster(admin, 0, st) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + // a roster below the roster cursor is a retransmit + e.roster(admin, 0, st) + eventually(t, func() bool { return e.counters().Retransmit == 1 }) + + // a valid chunk whose payload is not a roster changes nothing + e.publish(admin, bps.KindRoster, 1, []byte{0x0a, 0x02, 0x01, 0x02}) + eventually(t, func() bool { return e.counters().InvalidRoster == 1 }) + e.roster(admin, 1) + expectKind(t, sub, bps.KindRoster, e.principal, 1) + + // a late joiner receives the latest roster first + late := e.subscribe() + m := expectKind(t, late, bps.KindRoster, e.principal, 1) + if len(m.Roster) != 0 { + t.Fatalf("roster %x", m.Roster) + } +} + +func TestServiceKindsAdminOnly(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Identity: st.addr}) + var cue pb.Broadcast + if err := r.ReadMsg(&cue); err != nil { + t.Fatal(err) + } + auth := e.chunk(st.signer, bps.KindAuth, ack.Challenge, 0, nil) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}); err != nil { + t.Fatal(err) + } + payload, _ := bps.RosterPayload([][]byte{st.addr}) + ros := e.chunk(st.signer, bps.KindRoster, ack.Challenge, 1, payload) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_ROSTER, Challenge: ack.Challenge, Index: 1, Soc: ros}); err != nil { + t.Fatal(err) + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) && e.counters().WrongStream == 1 }) +} + +func TestEndOfStream(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + admin, _ := e.publisher() + sub := e.subscribe() + second, _ := e.pending() + e.publish(admin, bps.KindData, 0, []byte("last words")) + expectKind(t, sub, bps.KindData, e.principal, 0) + + e.publish(admin, bps.KindEOS, 0, nil) + // every stream is told — the sender and a pending one included — and closed + for _, s := range []bps.Session{sub, admin, second} { + expectKind(t, s, bps.KindEOS, e.principal, 0) + expectDone(t, s) + if !errors.Is(s.Err(), bps.ErrEndOfStream) { + t.Fatalf("got %v, want %v", s.Err(), bps.ErrEndOfStream) + } + } + eventually(t, func() bool { + _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}) + return !ok + }) + + // a channel the subscriber has seen end is not rejoined + pos := sub.Position() + if !pos.Ended { + t.Fatal("not ended") + } + c, _ := e.client(swarm.RandAddress(t)) + addr := make([]byte, 20) + _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: addr, Position: pos}) + if !errors.Is(err, bps.ErrChannelEnded) { + t.Fatalf("got %v, want %v", err, bps.ErrChannelEnded) + } + // a Join that does not know creates a fresh cohort + fresh := e.subscribe() + expectNoMessage(t, fresh) + if _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}); !ok { + t.Fatal("no fresh cohort") + } +} + +func TestPublisherStreamsBound(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxPublisherStreams: 2}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + sess := make([]bps.Session, 0, 3) + for range 3 { + s, _ := e.joinAs(st) + drain(s) + waitGranted(t, s) + e.publishAs(st, s, bps.KindAuth, 0, nil) + sess = append(sess, s) + time.Sleep(50 * time.Millisecond) + } + // the third auth resets the oldest + expectDone(t, sess[0]) + select { + case <-sess[1].Done(): + t.Fatal("second stream reset") + case <-sess[2].Done(): + t.Fatal("third stream reset") + default: + } +} + +// TestRosteredPastFullSet: a rostered identity that finds the fan-out set full +// is pending — the roster, nothing else — and authenticates as any grantee; +// one that does not is reset at the auth deadline, not blocklisted. +func TestRosteredPastFullSet(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxSubsPerCohort: 1, AuthWaitTimeout: 300 * time.Millisecond}) + st, idle := e.newSeat(), e.newSeat() + admin, _ := e.publisher() + drain(admin) + probe := e.subscribe() // fills the fan-out set + e.roster(admin, 0, st, idle) + expectKind(t, probe, bps.KindRoster, e.principal, 0) + + seatSess, _ := e.joinAs(st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + e.publishAs(st, seatSess, bps.KindData, 0, []byte("past the bound")) + expectKind(t, probe, bps.KindData, st.addr, 0) + + listener, overlay := e.joinAs(idle) + expectKind(t, listener, bps.KindRoster, e.principal, 0) + e.publish(admin, bps.KindData, 0, []byte("not for a pending stream")) + expectKind(t, probe, bps.KindData, e.principal, 0) + expectDone(t, listener) + if e.bl.blocklisted(overlay) || e.counters().AuthTimeout != 1 { + t.Fatal("a rostered pending stream times out without a blocklist") + } +} diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index 9ba6b51cdc0..a40c3e39789 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -86,18 +86,26 @@ const ( Kind_KIND_UNSPECIFIED Kind = 0 Kind_DATA Kind = 1 Kind_AUTH Kind = 2 + // a pending stream, or a spectator stream a delivered roster names; + // a heartbeat on a publisher stream; never delivered + Kind_EOS Kind = 3 + Kind_ROSTER Kind = 4 ) var Kind_name = map[int32]string{ 0: "KIND_UNSPECIFIED", 1: "DATA", 2: "AUTH", + 3: "EOS", + 4: "ROSTER", } var Kind_value = map[string]int32{ "KIND_UNSPECIFIED": 0, "DATA": 1, "AUTH": 2, + "EOS": 3, + "ROSTER": 4, } func (x Kind) String() string { @@ -275,6 +283,51 @@ func (m *Ack) GetChallenge() []byte { return nil } +// The payload of a ROSTER chunk. +type Roster struct { + Publishers [][]byte `protobuf:"bytes,1,rep,name=publishers,proto3" json:"publishers,omitempty"` +} + +func (m *Roster) Reset() { *m = Roster{} } +func (m *Roster) String() string { return proto.CompactTextString(m) } +func (*Roster) ProtoMessage() {} +func (*Roster) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{3} +} +func (m *Roster) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Roster) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Roster.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Roster) XXX_Merge(src proto.Message) { + xxx_messageInfo_Roster.Merge(m, src) +} +func (m *Roster) XXX_Size() int { + return m.Size() +} +func (m *Roster) XXX_DiscardUnknown() { + xxx_messageInfo_Roster.DiscardUnknown(m) +} + +var xxx_messageInfo_Roster proto.InternalMessageInfo + +func (m *Roster) GetPublishers() [][]byte { + if m != nil { + return m.Publishers + } + return nil +} + // Every frame after the handshake. soc is the index-th update of the session feed: // a single-owner chunk whose id is derived from kind, challenge and index (see bps.ID). type Broadcast struct { @@ -288,7 +341,7 @@ func (m *Broadcast) Reset() { *m = Broadcast{} } func (m *Broadcast) String() string { return proto.CompactTextString(m) } func (*Broadcast) ProtoMessage() {} func (*Broadcast) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{3} + return fileDescriptor_786299925f8760d5, []int{4} } func (m *Broadcast) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -352,40 +405,43 @@ func init() { proto.RegisterType((*CohortSpec)(nil), "bps.CohortSpec") proto.RegisterType((*Join)(nil), "bps.Join") proto.RegisterType((*Ack)(nil), "bps.Ack") + proto.RegisterType((*Roster)(nil), "bps.Roster") proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") } func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 424 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xc7, 0xe3, 0x0f, 0x4c, 0x32, 0x8d, 0xc2, 0x32, 0xaa, 0x50, 0x40, 0xad, 0x15, 0x85, 0x03, - 0x51, 0x90, 0x2a, 0x14, 0xce, 0x48, 0x38, 0xb1, 0x43, 0xdd, 0x54, 0x69, 0xb5, 0x76, 0x2e, 0x5c, - 0x22, 0x7f, 0xa9, 0x5d, 0x35, 0xec, 0xae, 0xe2, 0x45, 0x82, 0xb7, 0xe0, 0xb1, 0x38, 0xf6, 0xc8, - 0x11, 0x25, 0x2f, 0x82, 0xbc, 0x4e, 0x08, 0xe4, 0xb6, 0xf3, 0xff, 0xcf, 0xfc, 0xe6, 0x43, 0x0b, - 0xad, 0x54, 0x96, 0x17, 0x72, 0x2d, 0x94, 0x40, 0x2b, 0x95, 0x65, 0xff, 0x0b, 0xc0, 0x44, 0xdc, - 0x8b, 0xb5, 0x8a, 0x64, 0x91, 0xe1, 0x29, 0x3c, 0x51, 0x42, 0xb2, 0xac, 0x6b, 0xf4, 0x8c, 0x41, - 0x9b, 0xd6, 0x01, 0xbe, 0x85, 0xa7, 0x29, 0xe3, 0x39, 0xe3, 0x77, 0x5d, 0xb3, 0x67, 0x0c, 0x3a, - 0xa3, 0xe7, 0x17, 0x15, 0x25, 0xae, 0xcc, 0x71, 0x6d, 0xd0, 0x7d, 0x06, 0x9e, 0x41, 0x4b, 0xae, - 0x19, 0xcf, 0x98, 0x4c, 0x56, 0x5d, 0x4b, 0x63, 0x0e, 0x42, 0x7f, 0x06, 0xf6, 0x95, 0x60, 0x1c, - 0xdf, 0x80, 0x93, 0xe9, 0xb6, 0xba, 0xd3, 0xc9, 0xe8, 0x99, 0x26, 0x1e, 0x26, 0xa1, 0x3b, 0x1b, - 0x5f, 0x41, 0x93, 0xe5, 0x05, 0x57, 0x4c, 0x7d, 0xd7, 0xcd, 0xdb, 0xf4, 0x6f, 0xdc, 0xbf, 0x04, - 0xcb, 0xcb, 0x1e, 0xf0, 0x35, 0x38, 0xa5, 0x4a, 0xd4, 0xd7, 0x52, 0xb3, 0x3a, 0xa3, 0x13, 0xcd, - 0x8a, 0xb4, 0x44, 0x77, 0x56, 0x35, 0x56, 0x76, 0x9f, 0xac, 0x56, 0x05, 0xbf, 0x2b, 0x76, 0xa0, - 0x83, 0xd0, 0x97, 0xd0, 0x1a, 0xaf, 0x45, 0x92, 0x67, 0x49, 0xa9, 0x90, 0x80, 0x55, 0x8a, 0xfd, - 0x09, 0xaa, 0x27, 0x9e, 0x83, 0xfd, 0xc0, 0x78, 0xbe, 0xdb, 0xbe, 0xa5, 0xf9, 0x33, 0xc6, 0x73, - 0xaa, 0xe5, 0xff, 0xd9, 0xd6, 0x11, 0xbb, 0xba, 0x29, 0xe3, 0x79, 0xf1, 0xad, 0x6b, 0xf7, 0x8c, - 0x81, 0x4d, 0xeb, 0x60, 0xf8, 0x01, 0xda, 0xff, 0xde, 0x0f, 0xcf, 0xe1, 0x65, 0x7c, 0x73, 0x1b, - 0x4e, 0x96, 0xe3, 0x70, 0xee, 0x87, 0xf3, 0x4f, 0xcb, 0xc5, 0x3c, 0xba, 0x0d, 0x26, 0xe1, 0x34, - 0x0c, 0x7c, 0xd2, 0xc0, 0x0e, 0xc0, 0x34, 0x08, 0xfc, 0xa5, 0xce, 0x21, 0xf6, 0xf0, 0x23, 0x38, - 0xf5, 0x82, 0xf8, 0x02, 0x30, 0x8a, 0xbd, 0x78, 0x11, 0x1d, 0x55, 0x38, 0x60, 0xde, 0xcc, 0x88, - 0x81, 0x4d, 0xb0, 0xa7, 0x8b, 0xeb, 0x6b, 0x62, 0x62, 0x1b, 0x9a, 0x34, 0xb8, 0x0a, 0x26, 0x71, - 0xe0, 0x13, 0x6b, 0xf8, 0x0e, 0xec, 0x6a, 0x05, 0x3c, 0x05, 0x32, 0x0b, 0xe7, 0xfe, 0x51, 0x75, - 0x13, 0x6c, 0xdf, 0x8b, 0xbd, 0xba, 0xde, 0x5b, 0xc4, 0x97, 0xc4, 0x1c, 0x9f, 0xfd, 0xdc, 0xb8, - 0xc6, 0xe3, 0xc6, 0x35, 0x7e, 0x6f, 0x5c, 0xe3, 0xc7, 0xd6, 0x6d, 0x3c, 0x6e, 0xdd, 0xc6, 0xaf, - 0xad, 0xdb, 0xf8, 0x6c, 0xca, 0x34, 0x75, 0xf4, 0xa7, 0x7a, 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, - 0x30, 0x35, 0xa6, 0xe5, 0x61, 0x02, 0x00, 0x00, + // 463 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0xcf, 0x6e, 0xd3, 0x40, + 0x10, 0xc6, 0xe3, 0xd8, 0xb8, 0xc9, 0xd4, 0x0a, 0xcb, 0xa8, 0x42, 0x01, 0xb5, 0x56, 0x64, 0x0e, + 0x58, 0x41, 0xea, 0x21, 0x9c, 0x91, 0x70, 0xe2, 0x0d, 0x75, 0x53, 0x25, 0xd5, 0xda, 0xb9, 0x70, + 0x89, 0xfc, 0x4f, 0xcd, 0xaa, 0xc1, 0x5e, 0xd9, 0xae, 0x04, 0x6f, 0xc1, 0x63, 0x71, 0xec, 0x91, + 0x23, 0x4a, 0x5e, 0x04, 0x79, 0x9d, 0x92, 0x92, 0x9b, 0xe7, 0x9b, 0x99, 0xdf, 0x7c, 0xfe, 0xb4, + 0xd0, 0x8d, 0x44, 0x79, 0x29, 0x8a, 0xbc, 0xca, 0x51, 0x8d, 0x44, 0x69, 0x7d, 0x03, 0x98, 0xe4, + 0xeb, 0xbc, 0xa8, 0x7c, 0x91, 0xc6, 0x78, 0x06, 0x2f, 0xaa, 0x5c, 0xf0, 0xb8, 0xaf, 0x0c, 0x14, + 0xdb, 0x60, 0x4d, 0x81, 0x1f, 0xe0, 0x24, 0xe2, 0x59, 0xc2, 0xb3, 0xbb, 0x7e, 0x7b, 0xa0, 0xd8, + 0xbd, 0xd1, 0xab, 0xcb, 0x9a, 0x12, 0xd4, 0xcd, 0x71, 0xd3, 0x60, 0x4f, 0x13, 0x78, 0x0e, 0x5d, + 0x51, 0xf0, 0x2c, 0xe6, 0x22, 0xdc, 0xf4, 0x55, 0x89, 0x39, 0x08, 0xd6, 0x0c, 0xb4, 0xeb, 0x9c, + 0x67, 0xf8, 0x1e, 0xf4, 0x58, 0x9e, 0x95, 0x97, 0x4e, 0x47, 0x2f, 0x25, 0xf1, 0xe0, 0x84, 0xed, + 0xdb, 0xf8, 0x16, 0x3a, 0x3c, 0x49, 0xb3, 0x8a, 0x57, 0x3f, 0xe4, 0x71, 0x83, 0xfd, 0xab, 0xad, + 0x2b, 0x50, 0x9d, 0xf8, 0x1e, 0xdf, 0x81, 0x5e, 0x56, 0x61, 0xf5, 0x50, 0x4a, 0x56, 0x6f, 0x74, + 0x2a, 0x59, 0xbe, 0x94, 0xd8, 0xbe, 0x55, 0xdb, 0x8a, 0xd7, 0xe1, 0x66, 0x93, 0x66, 0x77, 0xe9, + 0x1e, 0x74, 0x10, 0x2c, 0x1b, 0x74, 0x96, 0x97, 0x55, 0x5a, 0xa0, 0x09, 0x20, 0x1e, 0xa2, 0x0d, + 0x2f, 0xd7, 0x69, 0x51, 0x03, 0x55, 0xdb, 0x60, 0xcf, 0x14, 0x4b, 0x40, 0x77, 0x5c, 0xe4, 0x61, + 0x12, 0x87, 0x65, 0x85, 0x04, 0xd4, 0x32, 0x7f, 0x0a, 0xab, 0xfe, 0xc4, 0x0b, 0xd0, 0xee, 0x79, + 0x96, 0xec, 0x73, 0xea, 0x4a, 0x27, 0x33, 0x9e, 0x25, 0x4c, 0xca, 0xff, 0xbb, 0x50, 0x8f, 0x5c, + 0xd4, 0xe9, 0xf3, 0x2c, 0x49, 0xbf, 0xf7, 0xb5, 0x81, 0x62, 0x6b, 0xac, 0x29, 0x86, 0x9f, 0xc0, + 0x78, 0x9e, 0x34, 0x5e, 0xc0, 0x9b, 0x60, 0x71, 0xeb, 0x4d, 0x56, 0x63, 0x6f, 0xee, 0x7a, 0xf3, + 0x2f, 0xab, 0xe5, 0xdc, 0xbf, 0xa5, 0x13, 0x6f, 0xea, 0x51, 0x97, 0xb4, 0xb0, 0x07, 0x30, 0xa5, + 0xd4, 0x5d, 0xc9, 0x19, 0xa2, 0x0d, 0x3f, 0x83, 0xde, 0x44, 0x81, 0xaf, 0x01, 0xfd, 0xc0, 0x09, + 0x96, 0xfe, 0xd1, 0x86, 0x0e, 0xed, 0xc5, 0x8c, 0x28, 0xd8, 0x01, 0x6d, 0xba, 0xbc, 0xb9, 0x21, + 0x6d, 0x34, 0xa0, 0xc3, 0xe8, 0x35, 0x9d, 0x04, 0xd4, 0x25, 0xea, 0x90, 0x82, 0x56, 0xff, 0x02, + 0x9e, 0x01, 0x99, 0x79, 0x73, 0xf7, 0x68, 0xbb, 0x03, 0x9a, 0xeb, 0x04, 0x4e, 0xb3, 0xef, 0x2c, + 0x83, 0x2b, 0xd2, 0xc6, 0x13, 0x50, 0xe9, 0xc2, 0x27, 0x2a, 0x02, 0xe8, 0x6c, 0xe1, 0x07, 0x94, + 0x11, 0x6d, 0x7c, 0xfe, 0x6b, 0x6b, 0x2a, 0x8f, 0x5b, 0x53, 0xf9, 0xb3, 0x35, 0x95, 0x9f, 0x3b, + 0xb3, 0xf5, 0xb8, 0x33, 0x5b, 0xbf, 0x77, 0x66, 0xeb, 0x6b, 0x5b, 0x44, 0x91, 0x2e, 0xdf, 0xe4, + 0xc7, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x50, 0xef, 0xea, 0xc9, 0xa0, 0x02, 0x00, 0x00, } func (m *CohortSpec) Marshal() (dAtA []byte, err error) { @@ -507,6 +563,38 @@ func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *Roster) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Roster) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Roster) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Publishers) > 0 { + for iNdEx := len(m.Publishers) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Publishers[iNdEx]) + copy(dAtA[i:], m.Publishers[iNdEx]) + i = encodeVarintBps(dAtA, i, uint64(len(m.Publishers[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *Broadcast) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -618,6 +706,21 @@ func (m *Ack) Size() (n int) { return n } +func (m *Roster) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Publishers) > 0 { + for _, b := range m.Publishers { + l = len(b) + n += 1 + l + sovBps(uint64(l)) + } + } + return n +} + func (m *Broadcast) Size() (n int) { if m == nil { return 0 @@ -1016,6 +1119,91 @@ func (m *Ack) Unmarshal(dAtA []byte) error { } return nil } +func (m *Roster) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Roster: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Roster: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Publishers", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Publishers = append(m.Publishers, make([]byte, postIndex-iNdEx)) + copy(m.Publishers[len(m.Publishers)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Broadcast) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index 286b58f6834..07281645da0 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -2,8 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Wire format of SWIP-74 (BPS-lite). Field and enum numbers are shared with -// SWIP-60, which extends these messages by adding fields and values only. +// Wire format of SWIP-74 (BPS-lite) with the admin's service kinds of SWIP-87 +// (explicit multi-publisher): EOS, ROSTER and the Roster payload. Field and enum +// numbers are shared with SWIP-60, which extends these messages by adding fields +// and values only. syntax = "proto3"; package bps; @@ -20,7 +22,7 @@ enum TopicBinding { message CohortSpec { bytes topic = 1; // 32 bytes TopicBinding binding = 2; // FEED_TOPIC - bytes principal = 3; // 20 bytes, required: the publisher + bytes principal = 3; // 20 bytes, required: the admin } // Peer -> broker: the first frame on a fresh stream. @@ -47,7 +49,15 @@ enum Kind { KIND_UNSPECIFIED = 0; // invalid on the wire DATA = 1; // an update of the stream's session feed AUTH = 2; // the empty chunk at index 0 of the session's AUTH feed: authenticates - // a pending stream, a heartbeat on a publisher stream, never delivered + // a pending stream, or a spectator stream a delivered roster names; + // a heartbeat on a publisher stream; never delivered + EOS = 3; // the empty chunk at index 0 of the admin's EOS feed: the channel is ended + ROSTER = 4; // from the admin: the publisher set as of this index; payload = Roster +} + +// The payload of a ROSTER chunk. +message Roster { + repeated bytes publishers = 1; // 20-byte addresses: the complete set excluding the admin } // Every frame after the handshake. soc is the index-th update of the session feed: From 3e760722d61381bde9dae59dbf7f65e2f3f0a280 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 22 Oct 2026 08:48:44 +0200 Subject: [PATCH 26/29] feat(bps): a forging broker is blocklisted for ten days, a named constant Co-Authored-By: Claude Opus 5.5 --- pkg/bps/bps.go | 10 +++++++--- pkg/bps/bps_test.go | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index ec396be5e0c..e4c8b65a4b3 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -61,6 +61,11 @@ var ( errChannelEnded = errors.New("bps: channel ended") ) +// ForgingBrokerBlocklistDuration is how long a subscriber avoids a broker that +// forwarded a delivery it should have refused — one that does not validate, or +// whose owner the roster does not admit — which no honest broker sends. +const ForgingBrokerBlocklistDuration = 10 * 24 * time.Hour + // Options are the broker's resource bounds. Zero values take the defaults. type Options struct { MaxSubsPerCohort int // subscriber streams per cohort @@ -329,13 +334,12 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { sess.grant() } - // a broker that forwards what it should have refused is avoided for good: - // blocklisted for the longest period there is, which is forever + // a broker that forwards what it should have refused is avoided for long sess.forged = func(err error) { if s.blocklister == nil { return } - if berr := s.blocklister.Blocklist(req.Broker, 0, err.Error()); berr != nil { + if berr := s.blocklister.Blocklist(req.Broker, ForgingBrokerBlocklistDuration, err.Error()); berr != nil { s.logger.Debug("blocklist broker failed", "broker", req.Broker, "error", berr) } } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index 7f91183346e..e58cb86f5cc 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -498,13 +498,13 @@ func TestInvalidDelivery(t *testing.T) { if err := sub.Err(); !errors.Is(err, bps.ErrInvalidDelivery) { t.Fatalf("got %v, want %v", err, bps.ErrInvalidDelivery) } - // the broker is blocklisted for the longest period there is: forever + // the broker is blocklisted for long eventually(t, func() bool { return bl.blocklisted(e.brokerAddr) }) bl.mu.Lock() d := bl.durations[e.brokerAddr.ByteString()] bl.mu.Unlock() - if d != 0 { - t.Fatalf("broker blocklisted for %v, want forever (0)", d) + if d != bps.ForgingBrokerBlocklistDuration { + t.Fatalf("broker blocklisted for %v, want %v", d, bps.ForgingBrokerBlocklistDuration) } } From cd76312b46ccf194a4568004101bf023f9641894 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 22 Oct 2026 08:48:44 +0200 Subject: [PATCH 27/29] feat(api): a subscriber passes back its roster and a cursor per grantee MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The subscribe socket takes roster=address:index,... — the roster it last received, each grantee with the lowest index of its updates to deliver — and rosterCursor, the lowest roster index to accept, next to the owner's cursor. The node admits by that roster until the broker delivers a later one, so a reconnecting dApp neither replays a grantee's updates nor takes an honest broker's re-sent roster for a rollback. Co-Authored-By: Claude Opus 5.5 --- openapi/Swarm.yaml | 17 +++++++++++ pkg/api/bps.go | 69 +++++++++++++++++++++++++++++++++++++++++---- pkg/api/bps_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 8858fadfe1e..63d10c05a62 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1100,6 +1100,23 @@ paths: type: integer required: false description: Lowest index of the owner's updates to deliver. Defaults to 0. + - in: query + name: roster + schema: + type: string + required: false + description: | + The roster the subscriber last received, with a cursor per grantee: + comma-separated `address:index` pairs, each the lowest index of that grantee's + updates to deliver (at most 186; not the owner). A dApp keeps it from the + deliveries it received and passes it back when it reconnects; it is the set + the node admits updates by until the broker delivers a later roster. + - in: query + name: rosterCursor + schema: + type: integer + required: false + description: Lowest roster index to accept — one past the index of the roster passed in `roster`. Defaults to 0. responses: "200": description: Establishes a WebSocket subscription for incoming broadcasts diff --git a/pkg/api/bps.go b/pkg/api/bps.go index 3d9ac59443e..b3fddb0a095 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -11,7 +11,9 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "net/http" + "strconv" "strings" "time" @@ -51,7 +53,7 @@ type bpsRequest struct { topic []byte broker swarm.Address identity []byte - cursor uint64 + position bps.Position // the subscriber's place on the channel: cursor, roster, rosterCursor } // bpsGrantedMessage tells a seat's dApp that a roster naming its identity has @@ -151,9 +153,11 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger } queries := struct { - Broker swarm.Address `map:"broker"` - Identity []byte `map:"identity" validate:"omitempty,len=20"` - Cursor uint64 `map:"cursor"` + Broker swarm.Address `map:"broker"` + Identity []byte `map:"identity" validate:"omitempty,len=20"` + Cursor uint64 `map:"cursor"` + Roster string `map:"roster"` + RosterCursor uint64 `map:"rosterCursor"` }{} if response := s.mapStructure(r.URL.Query(), &queries); response != nil { response("invalid query params", logger, w) @@ -172,15 +176,68 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger return bpsRequest{}, false } + pos, err := bpsPosition(paths.Owner.Bytes(), queries.Cursor, queries.Roster, queries.RosterCursor) + if err != nil { + jsonhttp.BadRequest(w, err.Error()) + return bpsRequest{}, false + } + return bpsRequest{ owner: paths.Owner, topic: paths.Topic, broker: queries.Broker, identity: queries.Identity, - cursor: queries.Cursor, + position: pos, }, true } +// bpsMaxRoster is the most addresses a roster can name: what fits in one chunk. +const bpsMaxRoster = 186 + +var errBPSInvalidRoster = errors.New("invalid roster") + +// bpsPosition builds the subscriber's place on the channel from the query: the +// owner's cursor, and the roster as address:cursor pairs — the grantees it +// names, each with the lowest index of its updates to deliver — at the roster +// cursor, the lowest roster index to accept. A dApp keeps them from the +// deliveries it received and passes them back when it reconnects. +func bpsPosition(owner []byte, cursor uint64, roster string, rosterCursor uint64) (bps.Position, error) { + pos := bps.Position{ + Cursors: map[string]uint64{string(owner): cursor}, + RosterCursor: rosterCursor, + } + if roster == "" { + return pos, nil + } + entries := strings.Split(roster, ",") + if len(entries) > bpsMaxRoster { + return pos, fmt.Errorf("%w: more than %d addresses", errBPSInvalidRoster, bpsMaxRoster) + } + for _, e := range entries { + addr, index, ok := strings.Cut(e, ":") + if !ok { + return pos, fmt.Errorf("%w: %q is not address:index", errBPSInvalidRoster, e) + } + a, err := hex.DecodeString(strings.TrimPrefix(addr, "0x")) + if err != nil || len(a) != common.AddressLength { + return pos, fmt.Errorf("%w: address %q", errBPSInvalidRoster, addr) + } + if bytes.Equal(a, owner) { + return pos, fmt.Errorf("%w: the owner is not on its roster", errBPSInvalidRoster) + } + n, err := strconv.ParseUint(index, 10, 64) + if err != nil { + return pos, fmt.Errorf("%w: index %q", errBPSInvalidRoster, index) + } + if _, dup := pos.Cursors[string(a)]; dup { + return pos, fmt.Errorf("%w: address %q twice", errBPSInvalidRoster, addr) + } + pos.Cursors[string(a)] = n + pos.Roster = append(pos.Roster, a) + } + return pos, nil +} + func (s *Service) bpsUpgrade(w http.ResponseWriter, r *http.Request, logger log.Logger) (*websocket.Conn, bool) { upgrader := websocket.Upgrader{ ReadBufferSize: swarm.ChunkSize, @@ -281,7 +338,7 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo Broker: req.broker, Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, Identity: req.identity, - Position: bps.Position{Cursors: map[string]uint64{string(req.owner.Bytes()): req.cursor}}, + Position: req.position, }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index 71e612a3d43..b89b26d084f 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -13,6 +13,7 @@ import ( "fmt" "net/http" "net/url" + "strings" "sync" "testing" "time" @@ -762,3 +763,69 @@ func TestBPSPublishSeatNotGranted(t *testing.T) { } expectClose(t, conn, api.BPSCloseInvalidMessage) } + +func TestBPSSubscribePosition(t *testing.T) { + t.Parallel() + seat1 := bytes.Repeat([]byte{0x11}, 20) + seat2 := bytes.Repeat([]byte{0x22}, 20) + + t.Run("cursor, roster and rosterCursor reach the join", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + q := "broker=" + f.broker.String() + "&identity=" + hex.EncodeToString(f.identity) + + "&cursor=7&rosterCursor=3&roster=" + hex.EncodeToString(seat1) + ":4,0x" + hex.EncodeToString(seat2) + ":0" + u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path("subscribe"), RawQuery: q} + conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + t.Fatal(err) + } + testutil.CleanupCloser(t, conn) + var req bps.JoinRequest + deadline := time.Now().Add(5 * time.Second) + for { + f.bps.mu.Lock() + req = f.bps.req + f.bps.mu.Unlock() + if req.Identity != nil || time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + pos := req.Position + if pos.RosterCursor != 3 || len(pos.Roster) != 2 || !bytes.Equal(pos.Roster[0], seat1) || !bytes.Equal(pos.Roster[1], seat2) { + t.Fatalf("roster %x at %d", pos.Roster, pos.RosterCursor) + } + if pos.Cursors[string(f.owner)] != 7 || pos.Cursors[string(seat1)] != 4 || pos.Cursors[string(seat2)] != 0 || len(pos.Cursors) != 3 { + t.Fatalf("cursors %v", pos.Cursors) + } + }) + + for _, tc := range []struct { + name, roster string + }{ + {"no index", hex.EncodeToString(seat1)}, + {"short address", "1122:4"}, + {"bad index", hex.EncodeToString(seat1) + ":x"}, + {"the owner", "OWNER:1"}, + {"twice", hex.EncodeToString(seat1) + ":1," + hex.EncodeToString(seat1) + ":2"}, + {"too many", strings.Repeat(hex.EncodeToString(seat1)+":1,", 186) + hex.EncodeToString(seat2) + ":1"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + roster := strings.ReplaceAll(tc.roster, "OWNER", hex.EncodeToString(f.owner)) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, f.path("subscribe")+"?broker="+f.broker.String()+"&identity="+hex.EncodeToString(f.identity)+"&roster="+roster, nil) + if err != nil { + t.Fatal(err) + } + resp, err := f.client.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status %d, want 400", resp.StatusCode) + } + }) + } +} From 5eea30d21b9dd544959eb2b343218e6b9a3ad0cd Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 29 Oct 2026 04:40:09 +0530 Subject: [PATCH 28/29] refactor(bps): the handler alone ends a stream; others ask by one-slot request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (acud): the broker's Close uses a quit channel; cohort.end is members, and shutdown goes; join returns a leave closure in place of the leave method; a member has a constructor and methods in place of stored closures and sync.Once; reclaim and the frame path use defer and one lock. A stream's handler owns it and is the only code that closes or resets it. Anyone else — a revocation, an EOS, a full queue, an eviction, a reclaim, Close — asks through one of the member's two one-slot channels: a send that never blocks, a repeat that is dropped, a channel never closed, so no ask can panic and the stream ends exactly once; a reset asked for takes precedence. Because an ask never blocks, it is made under the cohort's lock, and the streams to reset are no longer collected and reset after unlocking. The auth and flush deadlines are timers of the handler, so a stuck write cannot hold the flush deadline off. A full queue now counts one queue_reset per stream. Co-Authored-By: Claude Opus 5.5 --- pkg/bps/bps.go | 702 ++++++++++++++++++++++---------------- pkg/bps/flush_test.go | 4 + pkg/bps/lifecycle_test.go | 124 +++++++ 3 files changed, 529 insertions(+), 301 deletions(-) create mode 100644 pkg/bps/lifecycle_test.go diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index e4c8b65a4b3..c20ee265219 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -146,10 +146,11 @@ type Service struct { metrics metrics logger log.Logger + quit chan struct{} // closed by Close + mtx sync.Mutex cohorts map[string]*cohort // by canonical spec peers map[string]map[string]int // peer overlay -> cohort key -> streams - closed bool } // New returns a new bps Service. The blocklister may be nil. @@ -160,6 +161,7 @@ func New(streamer p2p.Streamer, blocklister p2p.Blocklister, broker bool, logger blocklister: blocklister, opts: o.withDefaults(), metrics: newMetrics(), + quit: make(chan struct{}), cohorts: make(map[string]*cohort), peers: make(map[string]map[string]int), logger: logger.WithName(loggerName).Register(), @@ -180,17 +182,24 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } } -// Close reclaims every cohort and resets their streams. +// Close refuses every later join, forgets every cohort and asks every stream +// in them to reset. A second call does nothing. func (s *Service) Close() error { s.mtx.Lock() - s.closed = true - cohorts := s.cohorts - s.cohorts = make(map[string]*cohort) - s.mtx.Unlock() - for _, co := range cohorts { - for _, m := range co.shutdown() { - m.reset() + defer s.mtx.Unlock() + select { + case <-s.quit: + return nil + default: + } + close(s.quit) + for _, co := range s.cohorts { + co.mtx.Lock() + s.removeCohort(co) + for _, m := range co.members() { + m.requestReset() } + co.mtx.Unlock() } return nil } @@ -584,83 +593,170 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er // re-marshal so that the cohort key ignores fields BPS-lite does not define join.Cohort = &pb.CohortSpec{Topic: join.Cohort.Topic, Binding: join.Cohort.Binding, Principal: join.Cohort.Principal} - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - co, m, err := s.join(p.Address, &join, func() { - cancel() - _ = stream.Reset() - }, func() { - // closing, not resetting, lets the frames already written arrive - _ = stream.Close() - cancel() - }) + co, m, leave, err := s.join(p.Address, &join) if err != nil { return s.refuse(ctx, w, stream, err) } // detach on every exit, normal or not - defer s.leave(p.Address, co, m) + defer leave() if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: m.challenge}); err != nil { - m.reset() + _ = stream.Reset() return fmt.Errorf("write ack: %w", err) } + return s.serve(ctx, p.Address, stream, w, r, co, m) +} - // the writer side: deliveries to a receiving stream, and the roster a - // pending stream is named in - writerDone := make(chan struct{}) - go func() { - defer close(writerDone) - for { - select { - case f := <-m.queue: - if err := w.WriteMsgWithContext(ctx, f); err != nil { - m.reset() - return - } - case <-m.ended: - // an EOS or a revocation: write what is queued — the EOS or the - // reduced roster last — then close; a peer that does not read - // within the flush deadline is reset - flushCtx, flushCancel := context.WithTimeout(ctx, s.opts.FlushTimeout) - defer flushCancel() - for { - select { - case f := <-m.queue: - if err := w.WriteMsgWithContext(flushCtx, f); err != nil { - m.reset() - return - } - default: - m.close() - return - } - } - case <-ctx.Done(): - return - } +// serve runs the two halves of an attached stream — the reader, handling the +// peer's frames, and the writer, delivering the stream's queue — and is the +// one place where the stream ends: it alone closes or resets it, on its own +// deadlines and on the requests anyone else makes of the member. It returns +// once both halves have stopped. +func (s *Service) serve(ctx context.Context, peer swarm.Address, stream p2p.Stream, w protobuf.Writer, r protobuf.Reader, co *cohort, m *member) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // each half closes its done channel when it stops, the reason left in its + // err; flush is closed here, once: write what is queued, then stop + flush := make(chan struct{}) + readDone, writeDone := make(chan struct{}), make(chan struct{}) + var readErr, writeErr error + go func() { defer close(readDone); readErr = s.read(ctx, r, co, m) }() + go func() { defer close(writeDone); writeErr = write(ctx, w, m.queue, flush) }() + readC := readDone // nil once seen, so that the closed channel is not seen again + + // the auth deadline of a stream admitted pending + var authC <-chan time.Time + if m.authDeadline { + t := time.NewTimer(s.opts.AuthWaitTimeout) + defer t.Stop() + authC = t.C + } + // the flush deadline, from the moment the stream is asked to close + var flushTimer *time.Timer + var flushC <-chan time.Time + defer func() { + if flushTimer != nil { + flushTimer.Stop() } }() - defer func() { <-writerDone }() - // the reader side: publications and authentications + // end closes or resets the stream and waits for both halves to stop + end := func(graceful bool) { + if graceful { + // closing, not resetting, lets the frames already written arrive + _ = stream.Close() + } else { + _ = stream.Reset() + } + cancel() + <-readDone + <-writeDone + } + for { - var f pb.Broadcast - if err := r.ReadMsgWithContext(ctx, &f); err != nil { - if ctx.Err() != nil { + // a reset asked for takes precedence over everything else + select { + case <-m.resetReq: + end(false) + return nil + default: + } + select { + case <-m.resetReq: + end(false) + return nil + + case <-m.closeReq: + // an EOS or a revocation: write the queue — the EOS or the reduced + // roster last — then close, within the flush deadline + if flushC == nil { + close(flush) + flushTimer = time.NewTimer(s.opts.FlushTimeout) + flushC = flushTimer.C + } + + case <-flushC: + // the peer did not take its queue in time, the write in progress + // included: the timer is not the writer's, so a stuck write cannot + // hold it off + end(false) + return nil + + case <-writeDone: + // the queue written after a close, or the writer failed + end(flushC != nil && writeErr == nil) + return nil + + case <-readC: + readC = nil + if flushC != nil { + // closing: nothing more is read anyway, the flush decides + continue + } + violation := errors.Is(readErr, errViolation) + if violation && s.blocklister != nil { + if berr := s.blocklister.Blocklist(peer, s.opts.ProtoBreachBlocklistDuration, readErr.Error()); berr != nil { + s.logger.Debug("blocklist failed", "peer_address", peer, "error", berr) + } + } + // a violation, or the peer went away + end(false) + if violation { + return readErr + } + return nil + + case <-authC: + authC = nil + if s.authExpired(peer, co, m) { + end(false) return nil } - m.reset() + + case <-ctx.Done(): + end(false) return nil } + } +} + +// read handles the peer's frames until the stream ends, and returns why: a +// violation, the peer going away, or ctx ending. +func (s *Service) read(ctx context.Context, r protobuf.Reader, co *cohort, m *member) error { + for { + var f pb.Broadcast + if err := r.ReadMsgWithContext(ctx, &f); err != nil { + return err + } if err := s.handleFrame(co, m, &f); err != nil { - if errors.Is(err, errViolation) && s.blocklister != nil { - if berr := s.blocklister.Blocklist(p.Address, s.opts.ProtoBreachBlocklistDuration, err.Error()); berr != nil { - s.logger.Debug("blocklist failed", "peer_address", p.Address, "error", berr) + return err + } + } +} + +// write delivers the stream's queue until ctx ends; once flush is closed, it +// writes what is still queued and returns nil. +func write(ctx context.Context, w protobuf.Writer, queue <-chan *pb.Broadcast, flush <-chan struct{}) error { + for { + select { + case f := <-queue: + if err := w.WriteMsgWithContext(ctx, f); err != nil { + return err + } + case <-flush: + for { + select { + case f := <-queue: + if err := w.WriteMsgWithContext(ctx, f); err != nil { + return err + } + default: + return nil } } - m.reset() - return err + case <-ctx.Done(): + return ctx.Err() } } } @@ -683,22 +779,11 @@ func (s *Service) refuse(ctx context.Context, w protobuf.Writer, stream p2p.Stre } // handleFrame validates a Broadcast arriving at the broker per SWIP-74 and -// SWIP-87 and accepts, drops or punishes it. A returned error resets the stream. +// SWIP-87 and accepts, drops or punishes it. A returned error is a violation. func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { principal := co.spec.Principal admin := bytes.Equal(m.identity, principal) - - co.mtx.Lock() - role := m.role - named, cued := co.rostered(m.identity), m.cued - var cursor uint64 - switch f.Kind { - case pb.Kind_DATA: - cursor = co.cursors[string(m.identity)] - case pb.Kind_ROSTER: - cursor = co.rosterCursor - } - co.mtx.Unlock() + role, named, cued, cursor := co.view(m, f.Kind) // 0. a stream being closed — revoked, or ended by an EOS — is not read: what // it still sends was sent before it read the roster or the EOS @@ -710,6 +795,8 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { // roster no longer names, raced its revocation: the stream is closed, the // AUTH dropped, nothing punished if role == roleSpectator && f.Kind == pb.Kind_AUTH && !named && cued { + co.mtx.Lock() + defer co.mtx.Unlock() s.revokeSpectator(co, m) return nil } @@ -739,7 +826,8 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { } // 5. the chunk must be its owner's at the id derived from the frame: the // stream's identity for DATA and AUTH, the principal for the service kinds; - // an AUTH or EOS is moreover the empty chunk at index 0 + // an AUTH or EOS is moreover the empty chunk at index 0. The signature check + // runs outside the lock, which is why the frame is applied in a second step. owner := m.identity if service(f.Kind) { owner = principal @@ -757,26 +845,32 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { return nil } } + s.apply(co, m, f, roster) + return nil +} - var resets, evicted, closes []*member +// apply applies a validated frame to the cohort. Every stream it ends, it ends +// by a request to the stream's handler, which never blocks, so it all runs +// under one lock. +func (s *Service) apply(co *cohort, m *member, f *pb.Broadcast, roster [][]byte) { co.mtx.Lock() + defer co.mtx.Unlock() + // a roster revoking the identity, or an EOS, was accepted since the checks if r := m.role; r == roleRevoked || r == roleEnded { - co.mtx.Unlock() s.dropClosing(co, r) - return nil + return } switch f.Kind { case pb.Kind_AUTH: switch { case m.role == roleSpectator && !co.rostered(m.identity): // revoked since the checks - co.mtx.Unlock() s.revokeSpectator(co, m) - return nil + return case m.role != rolePublisher: // the auth of a pending stream or of a spectator a roster names - resets, evicted = co.upgrade(m, s.opts.MaxPublisherStreams) + s.countQueueResets(co, co.upgrade(m, s.opts.MaxPublisherStreams)) } // on a publisher stream, a heartbeat co.inactiveSince = time.Now() @@ -786,59 +880,38 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { if f.Index < co.cursors[k] { // raced with another publisher stream of the same identity s.count(&co.counters.retransmit, "retransmit") - break + return } co.cursors[k] = f.Index + 1 co.inactiveSince = time.Now() s.metrics.Delivered.Inc() - resets = co.deliver(f, m) + s.countQueueResets(co, co.deliver(f, m)) case pb.Kind_ROSTER: if f.Index < co.rosterCursor { s.count(&co.counters.retransmit, "retransmit") - break + return } co.rosterCursor = f.Index + 1 co.inactiveSince = time.Now() - resets, closes = co.applyRoster(f, roster, m) + s.countQueueResets(co, co.applyRoster(f, roster, m)) case pb.Kind_EOS: // the channel is ended: every stream is told — pending ones too — and - // closed, and nothing more is read from any of them + // closed, and nothing more is read from any of them; the cohort is + // forgotten when the last of them has left, and a Join meanwhile + // creates a fresh one co.ended = true - resets = co.deliver(f, m) - for o := range co.members { - if o.role == rolePending && !o.enqueue(f) { - resets = append(resets, o) - } - } - closes = co.end() - for _, o := range closes { - if o.authTimer != nil { - o.authTimer.Stop() + overflow := co.deliver(f, m) + for _, o := range co.members() { + if o.role == rolePending && !o.enqueue(f) && o.requestReset() { + overflow++ } o.role = roleEnded + o.requestClose() } + s.countQueueResets(co, overflow) } - co.mtx.Unlock() - - // the resets of the overflowed streams, outside the lock (see deliver) - for _, r := range resets { - s.count(&co.counters.queueReset, "queue_reset") - r.reset() - } - for _, r := range evicted { - r.reset() - } - if f.Kind == pb.Kind_EOS { - s.endCohort(co, closes) - return nil - } - // the revoked streams are closed once the roster is written to them - for _, r := range closes { - r.end() - } - return nil } // mayEmit reports whether a stream of the role may send a frame of the kind @@ -865,55 +938,49 @@ func (s *Service) dropClosing(co *cohort, r role) { s.count(&co.counters.revoked, "revoked") } +// countQueueResets counts the streams reset because their queue was full. +func (s *Service) countQueueResets(co *cohort, n int) { + for range n { + s.count(&co.counters.queueReset, "queue_reset") + } +} + // revokeSpectator closes a spectator whose AUTH raced the roster revoking it. +// Must be called with co.mtx held. func (s *Service) revokeSpectator(co *cohort, m *member) { - co.mtx.Lock() if m.role == roleSpectator { m.role = roleRevoked co.spectators-- } - co.mtx.Unlock() s.count(&co.counters.revoked, "revoked") - m.end() -} - -// endCohort forgets a cohort ended by its admin's EOS and closes its streams -// once their queues — the EOS last — are written. -func (s *Service) endCohort(co *cohort, ms []*member) { - s.mtx.Lock() - if s.cohorts[co.key] == co { - co.mtx.Lock() - s.removeCohort(co) - co.mtx.Unlock() - } - s.mtx.Unlock() - for _, m := range ms { - m.end() - } + m.requestClose() } -// join attaches a stream to the cohort named by the spec, creating the -// cohort if no live one has it. An error wrapping errFull means a capacity -// bound was hit. reset ends the stream abruptly, close after its queue. -func (s *Service) join(peer swarm.Address, join *pb.Join, reset, closeFn func()) (*cohort, *member, error) { +// join attaches a stream to the cohort named by the spec, creating the cohort +// if no live one has it, and returns leave, which detaches the stream again: +// the stream's handler calls it once, when it returns. An error wrapping +// errFull means a capacity bound was hit. +func (s *Service) join(peer swarm.Address, join *pb.Join) (co *cohort, m *member, leave func(), err error) { key, err := join.Cohort.Marshal() if err != nil { - return nil, nil, fmt.Errorf("marshal cohort spec: %w", err) + return nil, nil, nil, fmt.Errorf("marshal cohort spec: %w", err) } - k := string(key) + k, pk := string(key), peer.ByteString() s.mtx.Lock() defer s.mtx.Unlock() - if s.closed { - return nil, nil, errSessionClosed + select { + case <-s.quit: + return nil, nil, nil, errSessionClosed + default: } - pc := s.peers[peer.ByteString()] + pc := s.peers[pk] if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerConnection { - return nil, nil, fmt.Errorf("too many cohorts per peer: %w", errFull) + return nil, nil, nil, fmt.Errorf("too many cohorts per peer: %w", errFull) } if pc[k] >= s.opts.MaxPeerStreamsPerCohort { - return nil, nil, fmt.Errorf("too many streams per peer per cohort: %w", errFull) + return nil, nil, nil, fmt.Errorf("too many streams per peer per cohort: %w", errFull) } co, ok := s.cohorts[k] @@ -929,15 +996,9 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset, closeFn func()) } if !ok { if len(s.cohorts) >= s.opts.MaxBrokerCohorts { - return nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) - } - co = &cohort{ - key: k, - spec: join.Cohort, - members: make(map[*member]struct{}), - cursors: make(map[string]uint64), - inactiveSince: time.Now(), + return nil, nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) } + co = newCohort(k, join.Cohort) co.inactivity = time.AfterFunc(s.opts.InactiveTimeout, func() { s.reclaim(co) }) s.cohorts[k] = co s.metrics.Cohorts.Set(float64(len(s.cohorts))) @@ -957,39 +1018,21 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset, closeFn func()) role = rolePending } if role == roleSpectator && full { - if len(co.members) == 0 { + if len(co.attached) == 0 { s.removeCohort(co) } - return nil, nil, fmt.Errorf("too many subscribers per cohort: %w", errFull) + return nil, nil, nil, fmt.Errorf("too many subscribers per cohort: %w", errFull) } challenge := make([]byte, ChallengeSize) if _, err := rand.Read(challenge); err != nil { - if len(co.members) == 0 { + if len(co.attached) == 0 { s.removeCohort(co) } - return nil, nil, fmt.Errorf("challenge: %w", err) - } - m := &member{ - role: role, - identity: join.Identity, - challenge: challenge, - queue: make(chan *pb.Broadcast, s.opts.QueueCap), - ended: make(chan struct{}), - } - var once, endOnce sync.Once - m.reset = func() { once.Do(reset) } - m.close = func() { once.Do(closeFn) } - // a stream being closed has the flush deadline to take its queue, the write - // in progress included; then it is reset, unless it closed in time - m.end = func() { - endOnce.Do(func() { - close(m.ended) - time.AfterFunc(s.opts.FlushTimeout, m.reset) - }) + return nil, nil, nil, fmt.Errorf("challenge: %w", err) } + m = newMember(role, join.Identity, challenge, s.opts.QueueCap) if role == rolePending { - m.authTimer = time.AfterFunc(s.opts.AuthWaitTimeout, func() { s.authTimeout(peer, co, m) }) // a rostered identity receives the roster that names it at once: its cue if !bytes.Equal(join.Identity, join.Cohort.Principal) { m.queue <- co.rosterFrame @@ -1004,45 +1047,39 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset, closeFn func()) m.cued = co.rostered(join.Identity) } } - co.members[m] = struct{}{} + co.attached[m] = struct{}{} if pc == nil { pc = make(map[string]int) - s.peers[peer.ByteString()] = pc + s.peers[pk] = pc } pc[k]++ - return co, m, nil -} -// leave detaches a stream. A cohort with no attached streams is reclaimed at once. -func (s *Service) leave(peer swarm.Address, co *cohort, m *member) { - s.mtx.Lock() - defer s.mtx.Unlock() + // leave detaches the stream: the peer's stream count, the member, and the + // cohort, once no stream is attached to it + leave = func() { + s.mtx.Lock() + defer s.mtx.Unlock() + if pc := s.peers[pk]; pc != nil { + if pc[k]--; pc[k] <= 0 { + delete(pc, k) + } + if len(pc) == 0 { + delete(s.peers, pk) + } + } - if pc := s.peers[peer.ByteString()]; pc != nil { - if pc[co.key]--; pc[co.key] <= 0 { - delete(pc, co.key) + co.mtx.Lock() + defer co.mtx.Unlock() + delete(co.attached, m) + if m.role == roleSpectator { + co.spectators-- } - if len(pc) == 0 { - delete(s.peers, peer.ByteString()) + if len(co.attached) == 0 && s.cohorts[k] == co { + s.removeCohort(co) } } - - co.mtx.Lock() - defer co.mtx.Unlock() - if _, ok := co.members[m]; !ok { - return - } - delete(co.members, m) - if m.authTimer != nil { - m.authTimer.Stop() - } - if m.role == roleSpectator { - co.spectators-- - } - if len(co.members) == 0 && s.cohorts[co.key] == co { - s.removeCohort(co) - } + return co, m, leave, nil } // removeCohort forgets the cohort. Must be called with s.mtx and co.mtx held. @@ -1053,41 +1090,33 @@ func (s *Service) removeCohort(co *cohort) { } // reclaim forgets a cohort on which no frame was accepted for the inactivity -// deadline, and resets every stream in it. +// deadline, and asks every stream in it to reset. func (s *Service) reclaim(co *cohort) { s.mtx.Lock() + defer s.mtx.Unlock() if s.cohorts[co.key] != co { - s.mtx.Unlock() return } + co.mtx.Lock() + defer co.mtx.Unlock() if idle := time.Since(co.inactiveSince); idle < s.opts.InactiveTimeout { co.inactivity.Reset(s.opts.InactiveTimeout - idle) - co.mtx.Unlock() - s.mtx.Unlock() return } s.removeCohort(co) - co.mtx.Unlock() - s.mtx.Unlock() - - s.logger.Debug("cohort reclaimed", "error", errCohortReclaimed) - for _, m := range co.shutdown() { - m.reset() + for _, m := range co.members() { + m.requestReset() } + s.logger.Debug("cohort reclaimed", "error", errCohortReclaimed) } -// authTimeout disconnects a pending stream that has not authenticated in time. -func (s *Service) authTimeout(peer swarm.Address, co *cohort, m *member) { - co.mtx.Lock() - pending := m.role == rolePending - if pending { - // an AUTH that arrives now is not accepted - m.role = roleRevoked - } - co.mtx.Unlock() - if !pending { - return +// authExpired ends a stream still pending at its auth deadline, and reports +// whether it was: a stream that authenticated, or began closing, before the +// deadline is left alone. +func (s *Service) authExpired(peer swarm.Address, co *cohort, m *member) bool { + if !co.expire(m) { + return false } s.count(&co.counters.authTimeout, "auth_timeout") // a rostered identity admitted past a full fan-out set that does not @@ -1097,7 +1126,7 @@ func (s *Service) authTimeout(peer swarm.Address, co *cohort, m *member) { s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) } } - m.reset() + return true } // Counters returns the counters of the live cohort named by the spec. @@ -1125,6 +1154,9 @@ const ( roleEnded // a stream of a cohort its admin's EOS ended: being closed, not read ) +// receives reports whether deliveries are enqueued on a stream of the role. +func (r role) receives() bool { return r == roleSpectator || r == rolePublisher } + type cohort struct { key string spec *pb.CohortSpec @@ -1137,10 +1169,32 @@ type cohort struct { rosterCursor uint64 // the lowest ROSTER index accepted next rosterFrame *pb.Broadcast // the latest accepted ROSTER frame: the first delivery inactiveSince time.Time - members map[*member]struct{} - spectators int // the fan-out set - authSeq uint64 // orders publisher streams by their auth - ended bool // the admin's EOS was accepted + attached map[*member]struct{} // the streams attached to the cohort + spectators int // the fan-out set + authSeq uint64 // orders publisher streams by their auth + ended bool // the admin's EOS was accepted +} + +// newCohort returns an empty cohort for the spec, keyed by its canonical +// serialisation. The caller arms its inactivity timer. +func newCohort(key string, spec *pb.CohortSpec) *cohort { + return &cohort{ + key: key, + spec: spec, + attached: make(map[*member]struct{}), + cursors: make(map[string]uint64), + inactiveSince: time.Now(), + } +} + +// members returns the streams attached to the cohort. Must be called with +// co.mtx held. +func (co *cohort) members() []*member { + ms := make([]*member, 0, len(co.attached)) + for m := range co.attached { + ms = append(ms, m) + } + return ms } // rostered reports whether the latest roster names the address. Must be @@ -1149,43 +1203,63 @@ func (co *cohort) rostered(a []byte) bool { return contains(co.roster, a) } -// receives reports whether deliveries are enqueued on a stream of the role. -func (r role) receives() bool { return r == roleSpectator || r == rolePublisher } +// view returns what the checks before the signature need of a stream: its +// role, whether the roster names its identity, whether a roster naming it was +// enqueued on it, and the cursor of the feed a frame of the kind is on. +func (co *cohort) view(m *member, kind pb.Kind) (r role, named, cued bool, cursor uint64) { + co.mtx.Lock() + defer co.mtx.Unlock() + switch kind { + case pb.Kind_DATA: + cursor = co.cursors[string(m.identity)] + case pb.Kind_ROSTER: + cursor = co.rosterCursor + } + return m.role, co.rostered(m.identity), m.cued, cursor +} + +// expire marks a stream still pending at its auth deadline as ending, so that +// an AUTH arriving now is not accepted, and reports whether it was pending. +func (co *cohort) expire(m *member) bool { + co.mtx.Lock() + defer co.mtx.Unlock() + if m.role != rolePending { + return false + } + m.role = roleRevoked + return true +} // deliver enqueues an accepted frame on every receiving stream — for DATA, -// every one not bound to the sender's identity — and returns the streams whose -// queue was full. A ROSTER or EOS reaches the sender too: where it sits in the -// sender's stream is where it took effect, and every receiver, the admin -// included, admits by the rosters it receives. The send is -// non-blocking, so that one slow receiver cannot stall the cohort; a stream -// whose queue is full is returned and reset by the caller after co.mtx is -// released — reset ends the stream, whose handler then calls leave, which -// takes s.mtx and co.mtx, so resetting under the lock would deadlock. Must be -// called with co.mtx held. -func (co *cohort) deliver(f *pb.Broadcast, sender *member) []*member { - var overflow []*member - for m := range co.members { - if !m.role.receives() || f.Kind == pb.Kind_DATA && bytes.Equal(m.identity, sender.identity) { +// every one not bound to the sender's identity — and returns how many streams +// were asked to reset because their queue was full. A ROSTER or EOS reaches +// the sender too: where it sits in the sender's stream is where it took +// effect, and every receiver, the admin included, admits by the rosters it +// receives. The send is non-blocking, so that one slow receiver cannot stall +// the cohort. Must be called with co.mtx held. +func (co *cohort) deliver(f *pb.Broadcast, sender *member) (overflow int) { + for m := range co.attached { + if !m.role.receives() || m.resetting || f.Kind == pb.Kind_DATA && bytes.Equal(m.identity, sender.identity) { continue } - if !m.enqueue(f) { - overflow = append(overflow, m) + if !m.enqueue(f) && m.requestReset() { + overflow++ } } return overflow } // upgrade makes a pending stream, or a spectator stream a roster names, a -// publisher stream, and returns the streams to reset: one whose queue was -// full, and the oldest publisher streams of the identity beyond the bound. -// Must be called with co.mtx held. -func (co *cohort) upgrade(m *member, maxStreams int) (resets, evicted []*member) { +// publisher stream; beyond the bound on the publisher streams one address may +// hold, the oldest of them is asked to reset. It returns 1 if the stream's +// queue could not take the latest roster, and 0 otherwise. Must be called with +// co.mtx held. +func (co *cohort) upgrade(m *member, maxStreams int) (overflow int) { if m.role == rolePending { - m.authTimer.Stop() // a stream that starts receiving gets the latest roster first, unless // it was given it as its cue - if co.rosterFrame != nil && m.given != co.rosterFrame && !m.enqueue(co.rosterFrame) { - resets = append(resets, m) + if co.rosterFrame != nil && m.given != co.rosterFrame && !m.enqueue(co.rosterFrame) && m.requestReset() { + overflow++ } } else { // a spectator leaves the fan-out set; it has the roster already @@ -1196,8 +1270,8 @@ func (co *cohort) upgrade(m *member, maxStreams int) (resets, evicted []*member) m.authSeq = co.authSeq var same []*member - for o := range co.members { - if o.role == rolePublisher && bytes.Equal(o.identity, m.identity) { + for o := range co.attached { + if o.role == rolePublisher && !o.resetting && bytes.Equal(o.identity, m.identity) { same = append(same, o) } } @@ -1208,24 +1282,24 @@ func (co *cohort) upgrade(m *member, maxStreams int) (resets, evicted []*member) oldest = i } } - evicted = append(evicted, same[oldest]) + same[oldest].requestReset() same = append(same[:oldest], same[oldest+1:]...) } - return resets, evicted + return overflow } // applyRoster makes an accepted roster the cohort's: it is delivered — to its // sender too — and every publisher stream of an identity it drops is revoked: // written the roster, then closed, what it sends meanwhile dropped; the // cursors of dropped identities are forgotten. A spectator it names may send -// its AUTH. It returns the streams to reset for a full queue and the revoked -// streams to close. Must be called with co.mtx held. -func (co *cohort) applyRoster(f *pb.Broadcast, roster [][]byte, sender *member) (resets, closes []*member) { +// its AUTH. It returns how many streams were asked to reset because their +// queue was full. Must be called with co.mtx held. +func (co *cohort) applyRoster(f *pb.Broadcast, roster [][]byte, sender *member) (overflow int) { principal := co.spec.Principal co.roster = roster co.rosterFrame = f - resets = co.deliver(f, sender) - for m := range co.members { + overflow = co.deliver(f, sender) + for m := range co.attached { if bytes.Equal(m.identity, principal) { continue } @@ -1236,23 +1310,24 @@ func (co *cohort) applyRoster(f *pb.Broadcast, roster [][]byte, sender *member) case m.role == rolePending && co.rostered(m.identity): // a rostered pending stream is given the latest roster m.given = f - if !m.enqueue(f) { - resets = append(resets, m) + if !m.enqueue(f) && m.requestReset() { + overflow++ } case m.role == rolePending: // revoked before its auth: told, then closed - m.authTimer.Stop() m.role = roleRevoked if !m.enqueue(f) { - resets = append(resets, m) + if m.requestReset() { + overflow++ + } continue } - closes = append(closes, m) + m.requestClose() case m.role == rolePublisher && !co.rostered(m.identity): // revoked: it was delivered this roster above; closed once that // is written m.role = roleRevoked - closes = append(closes, m) + m.requestClose() } } for k := range co.cursors { @@ -1260,41 +1335,66 @@ func (co *cohort) applyRoster(f *pb.Broadcast, roster [][]byte, sender *member) delete(co.cursors, k) } } - return resets, closes -} - -// end returns the cohort's streams for an EOS to close. Must be called with -// co.mtx held. -func (co *cohort) end() []*member { - ms := make([]*member, 0, len(co.members)) - for m := range co.members { - ms = append(ms, m) - } - return ms -} - -// shutdown returns the cohort's streams, for the caller to reset. -func (co *cohort) shutdown() []*member { - co.mtx.Lock() - defer co.mtx.Unlock() - return co.end() + return overflow } -// member is one stream attached to a cohort. role and authSeq are guarded by +// member is one stream attached to a cohort. Its handler owns the stream and +// is the only one that closes or resets it; anyone else asks, by +// requestClose or requestReset. The fields after the requests are guarded by // cohort.mtx. type member struct { + identity []byte + challenge []byte + authDeadline bool // admitted pending: the auth deadline applies + queue chan *pb.Broadcast + closeReq chan struct{} // one slot: write the queue, then close + resetReq chan struct{} // one slot: reset at once; takes precedence + role role - identity []byte - challenge []byte - queue chan *pb.Broadcast - ended chan struct{} // closed by an EOS: write the queue, then close - reset func() - close func() - end func() - authTimer *time.Timer authSeq uint64 cued bool // a roster naming the identity was enqueued on this spectator stream given *pb.Broadcast // the roster last enqueued on a rostered pending stream + resetting bool // a reset was asked for: nothing more is enqueued +} + +// newMember returns the member of a stream joining in the role, with its +// outbound queue and its two requests. +func newMember(r role, identity, challenge []byte, queueCap int) *member { + return &member{ + identity: identity, + challenge: challenge, + authDeadline: r == rolePending, + queue: make(chan *pb.Broadcast, queueCap), + closeReq: make(chan struct{}, 1), + resetReq: make(chan struct{}, 1), + role: r, + } +} + +// requestClose asks the stream's handler to write what is queued, then close +// the stream, within the flush deadline. It never blocks, and asking again +// changes nothing. +func (m *member) requestClose() { request(m.closeReq) } + +// requestReset asks the stream's handler to reset the stream at once, and +// keeps anything more from being enqueued on it. It never blocks; it reports +// whether this was the first ask. Must be called with cohort.mtx held. +func (m *member) requestReset() bool { + if m.resetting { + return false + } + m.resetting = true + request(m.resetReq) + return true +} + +// request fills a one-slot request channel, unless it is full already. The +// channel is never closed, so no ask can panic, however often it is made. +func request(c chan struct{}) { + select { + case c <- struct{}{}: + default: + } } // enqueue adds a frame to the stream's outbound queue, and reports false if diff --git a/pkg/bps/flush_test.go b/pkg/bps/flush_test.go index 1099c8474bb..dd1fa18f8ba 100644 --- a/pkg/bps/flush_test.go +++ b/pkg/bps/flush_test.go @@ -10,6 +10,7 @@ import ( "errors" "io" "sync" + "sync/atomic" "testing" "time" @@ -41,6 +42,7 @@ type flushStream struct { unblock chan struct{} // closed by Reset or Close reset, closed chan struct{} + resets, closes atomic.Int32 // every call, so that the test can tell the stream ended once stallOnce, unblockOnce sync.Once resetOnce, closedOnce sync.Once } @@ -76,12 +78,14 @@ func (s *flushStream) end() { } func (s *flushStream) Reset() error { + s.resets.Add(1) s.resetOnce.Do(func() { close(s.reset) }) s.end() return nil } func (s *flushStream) Close() error { + s.closes.Add(1) s.closedOnce.Do(func() { close(s.closed) }) s.end() return nil diff --git a/pkg/bps/lifecycle_test.go b/pkg/bps/lifecycle_test.go new file mode 100644 index 00000000000..2143348e2a9 --- /dev/null +++ b/pkg/bps/lifecycle_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "errors" + "sync" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// memberOf returns the cohort's member declaring identity. +func (e *flushEnv) memberOf(identity []byte) (*cohort, *member) { + e.t.Helper() + key, err := e.spec.Marshal() + if err != nil { + e.t.Fatal(err) + } + e.svc.mtx.Lock() + co := e.svc.cohorts[string(key)] + e.svc.mtx.Unlock() + if co == nil { + e.t.Fatal("no live cohort") + } + co.mtx.Lock() + defer co.mtx.Unlock() + for m := range co.attached { + if bytes.Equal(m.identity, identity) { + return co, m + } + } + e.t.Fatal("no such member") + return nil, nil +} + +// TestRequestsEndAStreamOnce: any number of requests to close and to reset a +// stream, from any number of goroutines, end it exactly once — closed or +// reset, never both — and never block or panic. +func TestRequestsEndAStreamOnce(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: 300 * time.Millisecond}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + co, m := e.memberOf(other) + + var wg sync.WaitGroup + for i := range 64 { + wg.Add(1) + go func() { + defer wg.Done() + if i%2 == 0 { + m.requestClose() + return + } + co.mtx.Lock() + m.requestReset() + co.mtx.Unlock() + }() + } + wg.Wait() + waitClosed(t, sub.done, "the stream's handler did not return") + if n := sub.st.resets.Load() + sub.st.closes.Load(); n != 1 { + t.Fatalf("the stream was ended %d times (resets %d, closes %d)", n, sub.st.resets.Load(), sub.st.closes.Load()) + } + // asking a stream that has ended changes nothing either + m.requestClose() + co.mtx.Lock() + m.requestReset() + co.mtx.Unlock() + notClosed(t, admin.st.reset, "the admin's stream reset") +} + +// TestLeave: when its streams end, a cohort leaves nothing behind — no member, +// no stream count of the peer, no cohort. +func TestLeave(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + + admin.mustSend(pb.Kind_EOS, 0, nil) + waitClosed(t, admin.done, "the admin's handler did not return") + waitClosed(t, sub.done, "the subscriber's handler did not return") + e.svc.mtx.Lock() + defer e.svc.mtx.Unlock() + if len(e.svc.cohorts) != 0 || len(e.svc.peers) != 0 { + t.Fatalf("left behind: %d cohorts, %d peers", len(e.svc.cohorts), len(e.svc.peers)) + } +} + +// TestServiceClose: Close resets every stream and refuses later joins, and a +// second Close does nothing. +func TestServiceClose(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + + if err := e.svc.Close(); err != nil { + t.Fatal(err) + } + if err := e.svc.Close(); err != nil { + t.Fatal(err) + } + waitClosed(t, admin.st.reset, "the admin's stream was not reset") + waitClosed(t, sub.st.reset, "the subscriber's stream was not reset") + waitClosed(t, sub.done, "the subscriber's handler did not return") + co, m, _, err := e.svc.join(swarm.RandAddress(t), &pb.Join{Cohort: e.spec, Identity: other}) + if !errors.Is(err, errSessionClosed) || co != nil || m != nil { + t.Fatalf("join after Close: got %v, want %v", err, errSessionClosed) + } +} From dc92712457f45a981cbb6556ec9e83e3481445b9 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 29 Oct 2026 08:52:56 +0530 Subject: [PATCH 29/29] refactor(bps): the session's read loop alone ends it; no sync.Once is left The client side gets the broker's treatment. A session's read loop is the one place where it ends: as it returns, it sets the reason, closes Messages and Done and ends the stream, once. Close only asks, by cancelling the context, which stops a read or a delivery in progress. The goroutine that reset the stream on that cancel, a second party ending it, is gone, and with it the sync.Once around the end. granted is closed by the constructor for the principal, and otherwise at the cue by the read loop alone, so its check cannot race its close. forged, a stored closure, is now a method, and the session has a constructor. A Close now closes the stream gracefully, as the read loop's exit always meant to: the watcher's reset used to win the race, and a reset can drop the last frames a publisher wrote before it closed. The test fakes count Resets and Closes under their mutex, and the API's fake session closes as the real one does, by a cancel. TestSessionEnd: a Close (64 at once), an EOS and the broker going away each end the stream once, closed or reset as due. Co-Authored-By: Claude Opus 5.5 --- pkg/api/bps_test.go | 22 +++-- pkg/bps/bps.go | 170 +++++++++++++++++++++----------------- pkg/bps/flush_test.go | 72 ++++++++++------ pkg/bps/lifecycle_test.go | 74 ++++++++++++++++- 4 files changed, 220 insertions(+), 118 deletions(-) diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index b89b26d084f..515ef2371a7 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -60,25 +60,25 @@ type fakeSession struct { granted chan struct{} done chan struct{} published chan published - closed chan struct{} + closed <-chan struct{} // closed by Close + cancel context.CancelFunc // Close, as the session's: a request, any number of times publishErr error - - doneOnce sync.Once - closeOnce sync.Once - err error + err error } func newFakeSession() *fakeSession { challenge := make([]byte, 32) copy(challenge, "bps-test-challenge") + ctx, cancel := context.WithCancel(context.Background()) return &fakeSession{ challenge: challenge, msgs: make(chan bps.Message), granted: closedChan(), done: make(chan struct{}), published: make(chan published, 8), - closed: make(chan struct{}), + closed: ctx.Done(), + cancel: cancel, } } @@ -104,16 +104,14 @@ func (f *fakeSession) Publish(_ context.Context, kind bps.Kind, index uint64, b } func (f *fakeSession) Close() error { - f.closeOnce.Do(func() { close(f.closed) }) + f.cancel() return nil } -// end simulates the broker stream ending with err. +// end simulates the broker stream ending with err; a test calls it once. func (f *fakeSession) end(err error) { - f.doneOnce.Do(func() { - f.err = err - close(f.done) - }) + f.err = err + close(f.done) } type bpsFixture struct { diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index c20ee265219..21d6b8ba124 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -326,67 +326,45 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { return nil, fmt.Errorf("challenge length %d: %w", len(ack.Challenge), errProtocolError) } - ctx, cancel := context.WithCancel(context.Background()) - sess := &session{ - ctx: ctx, - cancel: cancel, - w: w, - challenge: ack.Challenge, - spec: req.Spec, - identity: req.Identity, - pos: req.Position.clone(), - granted: make(chan struct{}), - rx: make(chan Message), - done: make(chan struct{}), - } - if bytes.Equal(req.Identity, req.Spec.Principal) { - sess.grant() - } - - // a broker that forwards what it should have refused is avoided for long - sess.forged = func(err error) { - if s.blocklister == nil { - return - } - if berr := s.blocklister.Blocklist(req.Broker, ForgingBrokerBlocklistDuration, err.Error()); berr != nil { - s.logger.Debug("blocklist broker failed", "broker", req.Broker, "error", berr) - } - } - - go sess.readLoop(stream, r, req.Spec) + sess := newSession(s, req, w, ack.Challenge) + go sess.readLoop(stream, r) return sess, nil } -// readLoop delivers the broker's frames until the stream ends. -func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader, spec CohortSpec) { - var readErr error - defer func() { - s.finish(readErr) - if readErr != nil && !errors.Is(readErr, errEndOfStream) && s.ctx.Err() == nil { - _ = stream.Reset() - } else { - _ = stream.FullClose() - } - }() - go func() { - <-s.ctx.Done() +// readLoop runs the session until its stream ends, and is the one place where +// the session ends: as it returns, it alone sets the reason, closes Messages +// and Done and ends the stream — closed after an EOS or a Close, reset on +// anything else. Close only asks, by cancelling the context, which stops a +// read or a delivery in progress. +func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader) { + defer s.cancel() + err := s.read(r) + s.err = err + close(s.rx) + close(s.done) + if err != nil && !errors.Is(err, errEndOfStream) && s.ctx.Err() == nil { _ = stream.Reset() - }() + return + } + _ = stream.FullClose() +} +// read delivers the broker's frames until the stream fails, an EOS arrives, a +// delivery does not verify or the session is closed, and returns why. +func (s *session) read(r protobuf.Reader) error { for { var f pb.Broadcast if err := r.ReadMsgWithContext(s.ctx, &f); err != nil { - readErr = err - return + return err } - msg, ok, err := s.accept(&f, spec) + msg, ok, err := s.accept(&f, s.spec) if err != nil { // a broker that forwards a chunk not valid for the spec, or by an // owner the roster does not admit, is misbehaving: the stream is ended - readErr = fmt.Errorf("%w: %w", errInvalidDelivery, err) - s.forged(readErr) - return + err = fmt.Errorf("%w: %w", errInvalidDelivery, err) + s.forged(err) + return err } if !ok { continue @@ -394,11 +372,10 @@ func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader, spec CohortSpec select { case s.rx <- msg: case <-s.ctx.Done(): - return + return nil } if msg.Kind == KindEOS { - readErr = errEndOfStream - return + return errEndOfStream } } } @@ -479,24 +456,48 @@ func contains(set [][]byte, a []byte) bool { return false } -// session is the concrete Session implementation. +// session is the concrete Session implementation. Its read loop owns its end +// (see readLoop). type session struct { - ctx context.Context - cancel context.CancelFunc - wmtx sync.Mutex - w protobuf.Writer - challenge []byte - spec CohortSpec - identity []byte - forged func(error) // called on a delivery the broker should have refused - mtx sync.Mutex - pos Position - granted chan struct{} - grantedOnce sync.Once - rx chan Message - done chan struct{} - err error - once sync.Once + svc *Service + broker swarm.Address + ctx context.Context + cancel context.CancelFunc // asks the read loop to stop + wmtx sync.Mutex + w protobuf.Writer + challenge []byte + spec CohortSpec + identity []byte + mtx sync.Mutex + pos Position + granted chan struct{} // closed at the cue, by the read loop; at once for the principal + rx chan Message // closed by the read loop as it returns + done chan struct{} // closed by the read loop as it returns + err error // set by the read loop before done is closed +} + +// newSession returns the session of a stream joined at the broker; the +// principal's may authenticate at once. +func newSession(svc *Service, req JoinRequest, w protobuf.Writer, challenge []byte) *session { + ctx, cancel := context.WithCancel(context.Background()) + s := &session{ + svc: svc, + broker: req.Broker, + ctx: ctx, + cancel: cancel, + w: w, + challenge: challenge, + spec: req.Spec, + identity: req.Identity, + pos: req.Position.clone(), + granted: make(chan struct{}), + rx: make(chan Message), + done: make(chan struct{}), + } + if bytes.Equal(req.Identity, req.Spec.Principal) { + close(s.granted) + } + return s } func (s *session) Challenge() []byte { return s.challenge } @@ -504,6 +505,9 @@ func (s *session) Messages() <-chan Message { return s.rx } func (s *session) Granted() <-chan struct{} { return s.granted } func (s *session) Done() <-chan struct{} { return s.done } func (s *session) Err() error { return s.err } + +// Close asks the read loop to end the session. It never blocks, and may be +// called any number of times, from any goroutine. func (s *session) Close() error { s.cancel() return nil @@ -515,7 +519,26 @@ func (s *session) Position() Position { return s.pos.clone() } -func (s *session) grant() { s.grantedOnce.Do(func() { close(s.granted) }) } +// grant closes granted, the cue to authenticate, unless it is closed already. +// Only the read loop calls it, so that the check and the close cannot race. +func (s *session) grant() { + select { + case <-s.granted: + default: + close(s.granted) + } +} + +// forged avoids, for long, a broker that forwarded a delivery it should have +// refused. +func (s *session) forged(err error) { + if s.svc.blocklister == nil { + return + } + if berr := s.svc.blocklister.Blocklist(s.broker, ForgingBrokerBlocklistDuration, err.Error()); berr != nil { + s.svc.logger.Debug("blocklist broker failed", "broker", s.broker, "error", berr) + } +} func (s *session) Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error { principal := bytes.Equal(s.identity, s.spec.Principal) @@ -546,15 +569,6 @@ func (s *session) Publish(ctx context.Context, kind Kind, index uint64, soc []by return nil } -// finish records the terminal error and signals stream completion exactly once. -func (s *session) finish(err error) { - s.once.Do(func() { - s.err = err - close(s.rx) - close(s.done) - }) -} - // validateJoin rejects a Join whose spec has a value outside SWIP-74 or whose // identity is not 20 bytes. func validateJoin(join *pb.Join) error { diff --git a/pkg/bps/flush_test.go b/pkg/bps/flush_test.go index dd1fa18f8ba..ca8fa05acef 100644 --- a/pkg/bps/flush_test.go +++ b/pkg/bps/flush_test.go @@ -10,7 +10,6 @@ import ( "errors" "io" "sync" - "sync/atomic" "testing" "time" @@ -27,24 +26,23 @@ import ( var errFlushStreamReset = errors.New("flush stream reset") -// flushStream is a broker-side stream whose peer is driven by the test: the -// peer's frames are written into a pipe the broker reads, and the broker's -// writes land in a buffer — or, once stalled, block as on a transport whose -// peer has stopped reading, until the stream is reset or closed. +// flushStream is a stream whose far end the test drives: the far end's frames +// are written into a pipe this end reads, and this end's writes land in a +// buffer — or, once stalled, block as on a transport whose far end has stopped +// reading, until the stream is reset or closed. It counts every Reset and +// Close, so that a test can tell the stream ended once. type flushStream struct { r *io.PipeReader pw *io.PipeWriter - mu sync.Mutex - out bytes.Buffer - stall bool - stalled chan struct{} // closed when a write blocks - unblock chan struct{} // closed by Reset or Close - - reset, closed chan struct{} - resets, closes atomic.Int32 // every call, so that the test can tell the stream ended once - stallOnce, unblockOnce sync.Once - resetOnce, closedOnce sync.Once + mu sync.Mutex + out bytes.Buffer + stall bool + blocked int // writes that blocked + stalled chan struct{} // closed when the first write blocks + resets, closes int + reset, closed chan struct{} // closed at the first Reset, the first Close + unblock chan struct{} // closed when the stream first ends, either way } func newFlushStream() *flushStream { @@ -52,9 +50,9 @@ func newFlushStream() *flushStream { return &flushStream{ r: r, pw: pw, stalled: make(chan struct{}), - unblock: make(chan struct{}), reset: make(chan struct{}), closed: make(chan struct{}), + unblock: make(chan struct{}), } } @@ -66,31 +64,53 @@ func (s *flushStream) Write(p []byte) (int, error) { defer s.mu.Unlock() return s.out.Write(p) } + s.blocked++ + if s.blocked == 1 { + close(s.stalled) + } s.mu.Unlock() - s.stallOnce.Do(func() { close(s.stalled) }) <-s.unblock return 0, errFlushStreamReset } -func (s *flushStream) end() { - s.unblockOnce.Do(func() { close(s.unblock) }) - _ = s.r.Close() -} - func (s *flushStream) Reset() error { - s.resets.Add(1) - s.resetOnce.Do(func() { close(s.reset) }) + s.mu.Lock() + defer s.mu.Unlock() + s.resets++ + if s.resets == 1 { + close(s.reset) + } s.end() return nil } func (s *flushStream) Close() error { - s.closes.Add(1) - s.closedOnce.Do(func() { close(s.closed) }) + s.mu.Lock() + defer s.mu.Unlock() + s.closes++ + if s.closes == 1 { + close(s.closed) + } s.end() return nil } +// end unblocks writes and reads when the stream first ends. Must be called +// with s.mu held. +func (s *flushStream) end() { + if s.resets+s.closes == 1 { + close(s.unblock) + _ = s.r.Close() + } +} + +// ended reports how many times the stream was reset and closed. +func (s *flushStream) ended() (resets, closes int) { + s.mu.Lock() + defer s.mu.Unlock() + return s.resets, s.closes +} + func (s *flushStream) FullClose() error { return s.Close() } func (s *flushStream) Headers() p2p.Headers { return nil } func (s *flushStream) ResponseHeaders() p2p.Headers { return nil } diff --git a/pkg/bps/lifecycle_test.go b/pkg/bps/lifecycle_test.go index 2143348e2a9..27dd287304d 100644 --- a/pkg/bps/lifecycle_test.go +++ b/pkg/bps/lifecycle_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" "github.com/ethersphere/bee/v2/pkg/swarm" ) @@ -67,8 +68,8 @@ func TestRequestsEndAStreamOnce(t *testing.T) { } wg.Wait() waitClosed(t, sub.done, "the stream's handler did not return") - if n := sub.st.resets.Load() + sub.st.closes.Load(); n != 1 { - t.Fatalf("the stream was ended %d times (resets %d, closes %d)", n, sub.st.resets.Load(), sub.st.closes.Load()) + if resets, closes := sub.st.ended(); resets+closes != 1 { + t.Fatalf("the stream was ended %d times (resets %d, closes %d)", resets+closes, resets, closes) } // asking a stream that has ended changes nothing either m.requestClose() @@ -122,3 +123,72 @@ func TestServiceClose(t *testing.T) { t.Fatalf("join after Close: got %v, want %v", err, errSessionClosed) } } + +// TestSessionEnd: a session's read loop alone ends its stream, once — closed +// after a Close or an EOS, reset when the broker goes away — however many +// Closes are asked for, from however many goroutines, before or after. +func TestSessionEnd(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + end func(t *testing.T, sess *session, broker *peer) + err error // the session's Err, when it is checked + closed bool // the stream is closed, not reset + }{ + {"a Close closes the stream", func(_ *testing.T, sess *session, _ *peer) { + var wg sync.WaitGroup + for range 64 { + wg.Add(1) + go func() { + defer wg.Done() + _ = sess.Close() + }() + } + wg.Wait() + }, nil, true}, + {"an EOS closes the stream", func(t *testing.T, sess *session, broker *peer) { + t.Helper() + broker.mustSend(pb.Kind_EOS, 0, nil) + if msg := <-sess.Messages(); msg.Kind != KindEOS { + t.Errorf("got %v, want the EOS", msg.Kind) + } + }, ErrEndOfStream, true}, + {"the broker going away resets the stream", func(_ *testing.T, _ *session, broker *peer) { + _ = broker.st.pw.Close() + }, nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{}) + st := newFlushStream() + w, r := protobuf.NewWriterAndReader(st) + _, identity := newSigner(t) + spec := CohortSpec{Topic: e.spec.Topic, Principal: e.spec.Principal} + sess := newSession(e.svc, JoinRequest{Spec: spec, Identity: identity}, w, make([]byte, ChallengeSize)) + loop := make(chan struct{}) + go func() { + defer close(loop) + sess.readLoop(st, r) + }() + + // the test is the broker: it writes the admin's frames, signed under + // a publisher stream's challenge, into the session's stream + broker := &peer{e: e, st: st, w: protobuf.NewWriter(st.pw), signer: e.admin, challenge: make([]byte, ChallengeSize)} + tc.end(t, sess, broker) + waitClosed(t, loop, "the read loop did not return") + _ = sess.Close() // asked after the end, it changes nothing + + resets, closes := st.ended() + if resets+closes != 1 || (closes == 1) != tc.closed { + t.Fatalf("resets %d, closes %d; want one, closed %t", resets, closes, tc.closed) + } + waitClosed(t, sess.Done(), "Done was not closed") + if _, ok := <-sess.Messages(); ok { + t.Fatal("Messages was not closed") + } + if tc.err != nil && !errors.Is(sess.Err(), tc.err) { + t.Fatalf("Err: got %v, want %v", sess.Err(), tc.err) + } + }) + } +}