diff --git a/cmd/bee/cmd/cmd.go b/cmd/bee/cmd/cmd.go index c28ee2a18b7..90dda83e0bf 100644 --- a/cmd/bee/cmd/cmd.go +++ b/cmd/bee/cmd/cmd.go @@ -62,6 +62,7 @@ const ( optionNameChequebookVerification = "chequebook-verification" optionNameChequebookMinBalance = "chequebook-min-balance" optionNameFullNode = "full-node" + optionNameBpsBroker = "bps-broker" optionNameLightNodeLimit = "light-node-limit" optionNamePostageContractAddress = "postage-stamp-address" optionNamePostageContractStartBlock = "postage-stamp-start-block" @@ -365,6 +366,7 @@ func (c *command) setAllFlags(cmd *cobra.Command) { cmd.Flags().Bool(optionNameChequebookVerification, false, "reject full-node hive/handshake records that carry no chequebook address") cmd.Flags().String(optionNameChequebookMinBalance, "110000000000000000", "minimum chequebook token balance required for verification, in token small units (default 11 BZZ)") cmd.Flags().Bool(optionNameFullNode, false, "cause the node to start in full mode") + cmd.Flags().Bool(optionNameBpsBroker, false, "enable the bps broker (requires full node)") cmd.Flags().Int(optionNameLightNodeLimit, 100, "light node limit") cmd.Flags().String(optionNamePostageContractAddress, "", "postage stamp contract address") cmd.Flags().Uint64(optionNamePostageContractStartBlock, 0, "postage stamp contract start block number") diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index 12b82d6b744..4fee4dfa8be 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -325,6 +325,7 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo AutoTLSDomain: c.config.GetString(optionAutoTLSDomain), AutoTLSRegistrationEndpoint: c.config.GetString(optionAutoTLSRegistrationEndpoint), FullNodeMode: fullNode, + BpsBroker: c.config.GetBool(optionNameBpsBroker), LightNodeLimit: c.config.GetInt(optionNameLightNodeLimit), Logger: logger, MinimumGasTipCap: c.config.GetUint64(optionNameMinimumGasTipCap), diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index cce68073b91..e6b9eaaaaf5 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1049,6 +1049,119 @@ paths: default: description: Default response + "/bps/feeds/subscribe/{principal}/{topic}": + get: + summary: Join a BPS cohort, to listen or to publish + description: | + WebSocket. The node joins the BPS cohort `{topic, FEED_TOPIC, principal}` (SWIP-74, with + the multi-publisher mode of SWIP-87) at `broker`, declaring `owner` if given, which settles + what the stream is, as it does on the wire. `owner` the principal's: a publisher stream + once it has sent its AUTH. Any other: a listener, which receives from the start and for + as long as it likes, and which may publish once the principal's roster names it (a seat). + None: a listener that can never publish, even if the key behind it is on the roster. + + 1. Server sends `{"challenge":"","broker":"","dataTopic":"","authTopic":""}`, + and to the principal also `"rosterTopic"` and `"eosTopic"`. A stream with no owner is + sent none of this: its socket carries deliveries alone. + `dataTopic` is `keccak256(topic | challenge)`, the session feed for updates; the + others are `keccak256("bps-service:v1" | kind | topic | challenge)` with kind 0x02 + (AUTH), 0x04 (ROSTER), 0x03 (EOS). + 2. Server sends every verified delivery as a binary frame + `kind (1) | owner (20) | challenge (32) | index (8, big-endian) | soc`, where `soc` is + the full single-owner chunk (`id | signature | span | payload`) signed by `owner` with + `id = keccak256(sessionTopic | index)`. `kind` is 1 (DATA): an update, by the + principal or by an address on its latest roster; 4 (ROSTER): the principal's publisher set, + payload a `Roster` protobuf of 20-byte addresses; or 3 (EOS): the principal ended the + channel, after which the socket closes with 4004. `sessionTopic` is as in step 1, under + the publishing stream's `challenge`. Deliveries that do not verify, by an owner the + roster does not admit, or below the stream's cursor are dropped. A publisher does not + get its own updates back; the principal does get its own rosters and EOS back. + 3. Once a roster naming a seat has reached its stream, the server sends the text frame + `granted`. The seat may then send its AUTH, which upgrades the stream in place. A listener + that sends anything before that is closed with 4002. A seat whose address a later roster + drops finds the socket closed after that roster (4003). + 4. To publish, the client sends binary frames `kind (1) | index (8, big-endian) | soc`, where + `soc` is a single-owner chunk with `id = keccak256(sessionTopic | index)`, signed by + `owner` for kind 1 (DATA) and 2 (AUTH) and by `principal` for kind 4 (ROSTER) and 3 (EOS), + which only the principal may send. The first frame must be the AUTH — the empty chunk at + index 0 of `authTopic`, which authenticates the stream; the principal's must arrive within 25 + seconds of the connection, and a seat has no such deadline. Sent again later it is a + heartbeat. DATA frames are updates of `dataTopic`. A ROSTER carries a `Roster` protobuf + naming every address that may publish besides the principal, full state at each index, the + principal and repeated addresses not among them; an EOS — the empty chunk at index 0 of + `eosTopic` — ends the channel. A principal that has a roster MUST publish it again, at its + next roster index, right after its AUTH: the broker may hold none, after an inactivity + reclaim or a restart. Indexes must not be reused across sessions; frames below the + broker's cursor are dropped silently. + + Close codes: 4001 invalid soc (an AUTH or EOS at another index or with a payload + included), 4002 invalid message (wrong frame type, a frame before the grant or after a + revocation, any frame on a stream with no owner, any frame before the AUTH, unknown kind, a ROSTER or EOS from a seat, an index + of 2^64-1, a roster that does not decode, or that names the principal or an address twice), + 4003 broker stream closed, 4004 end of stream, 4005 the principal's AUTH did not arrive in time, + 1001 the node is shutting down, 1009 a frame over the size limit. + tags: + - BPS + parameters: + - in: path + name: principal + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: true + description: Ethereum address of the cohort's principal, its admin, the only one who can change the roster or end the channel. A seat signs its own updates on its own feed, so this says whose channel the stream joins, and `owner` who is speaking. + - in: path + name: topic + schema: + $ref: "SwarmCommon.yaml#/components/schemas/HexString" + required: true + description: 32-byte cohort topic + - in: query + name: broker + schema: + $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" + required: true + description: Overlay address of the broker node. Must not be this node. + - in: query + name: owner + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: false + description: The owner of the stream's frames, the wallet address signing its feed update SOCs. The principal's to publish as the principal; any other to listen, and to publish once the principal's roster names it. Leave it out to only listen; such a stream can never publish, even if the key behind it is on the roster. + - in: query + name: cursor + schema: + type: integer + required: false + description: Lowest index of the principal's updates to deliver. Defaults to 0. + - in: query + name: roster + schema: + type: string + required: false + description: | + The roster the client last received, with a cursor per grantee: + comma-separated `address:index` pairs, each the lowest index of that grantee's + updates to deliver (at most 186; not the principal). A dApp keeps it from the + deliveries it received and passes it back when it reconnects; it is the set + the node admits updates by until the broker delivers a later roster. + - in: query + name: rosterCursor + schema: + type: integer + required: false + description: Lowest roster index to accept — one past the index of the roster passed in `roster`. Defaults to 0. + responses: + "200": + description: Establishes the WebSocket + "400": + $ref: "SwarmCommon.yaml#/components/responses/400" + "500": + $ref: "SwarmCommon.yaml#/components/responses/500" + "503": + description: BPS service is not configured + default: + description: Default response + "/soc/{owner}/{id}": post: summary: Upload a Single Owner Chunk diff --git a/packaging/bee.yaml b/packaging/bee.yaml index 4c91efae60d..85ab952a244 100644 --- a/packaging/bee.yaml +++ b/packaging/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/packaging/docker/env b/packaging/docker/env index 87043ca4f39..217631de5e9 100644 --- a/packaging/docker/env +++ b/packaging/docker/env @@ -31,6 +31,8 @@ # BEE_BOOTNODE=[/dnsaddr/mainnet.ethswarm.org] ## cause the node to always accept incoming connections (default false) # BEE_BOOTNODE_MODE=false +## enable the bps broker (requires full node) +# BEE_BPS_BROKER=false ## bzz token contract address # BEE_BZZ_TOKEN_ADDRESS= ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes (default 1000000) diff --git a/packaging/homebrew-amd64/bee.yaml b/packaging/homebrew-amd64/bee.yaml index f343dec9458..e53109ef03a 100644 --- a/packaging/homebrew-amd64/bee.yaml +++ b/packaging/homebrew-amd64/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/packaging/homebrew-arm64/bee.yaml b/packaging/homebrew-arm64/bee.yaml index 4bf3e8e117e..479a57db33d 100644 --- a/packaging/homebrew-arm64/bee.yaml +++ b/packaging/homebrew-arm64/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/packaging/scoop/bee.yaml b/packaging/scoop/bee.yaml index 2b5f6a94db0..ac971e029c5 100644 --- a/packaging/scoop/bee.yaml +++ b/packaging/scoop/bee.yaml @@ -19,6 +19,8 @@ # bootnode: ["/dnsaddr/mainnet.ethswarm.org"] ## cause the node to always accept incoming connections # bootnode-mode: false +## enable the bps broker (requires full node) +# bps-broker: false ## bzz token contract address # bzz-token-address: "" ## cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes diff --git a/pkg/api/api.go b/pkg/api/api.go index 38ac92c048b..fce6a7821e5 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/accesscontrol" "github.com/ethersphere/bee/v2/pkg/accounting" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/pipeline" @@ -166,6 +167,7 @@ type Service struct { resolver resolver.Interface pss pss.Interface gsoc gsoc.Listener + bps bps.Interface steward steward.Interface logger log.Logger loggerV1 log.Logger @@ -285,6 +287,7 @@ type ExtraOptions struct { Resolver resolver.Interface Pss pss.Interface Gsoc gsoc.Listener + Bps bps.Interface FeedFactory feeds.Factory Post postage.Service AccessControl accesscontrol.Controller @@ -368,6 +371,7 @@ func (s *Service) Configure(signer crypto.Signer, tracer *tracing.Tracer, o Opti s.resolver = e.Resolver s.pss = e.Pss s.gsoc = e.Gsoc + s.bps = e.Bps s.feedFactory = e.FeedFactory s.post = e.Post s.accesscontrol = e.AccessControl diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 02de8ce6d68..ba9fdf3630d 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -29,6 +29,7 @@ import ( mockac "github.com/ethersphere/bee/v2/pkg/accesscontrol/mock" accountingmock "github.com/ethersphere/bee/v2/pkg/accounting/mock" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/pipeline" @@ -96,6 +97,7 @@ type testServerOptions struct { Resolver resolver.Interface Pss pss.Interface Gsoc gsoc.Listener + Bps bps.Interface WsPath string WsPingPeriod time.Duration Logger log.Logger @@ -210,6 +212,7 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket. Resolver: o.Resolver, Pss: o.Pss, Gsoc: o.Gsoc, + Bps: o.Bps, FeedFactory: o.Feeds, Post: o.Post, AccessControl: o.AccessControl, diff --git a/pkg/api/bps.go b/pkg/api/bps.go new file mode 100644 index 00000000000..02b8502a5fd --- /dev/null +++ b/pkg/api/bps.go @@ -0,0 +1,438 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package api + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/jsonhttp" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" +) + +const ( + bpsCloseInvalidSOC = 4001 // a chunk that does not validate under the session challenge + bpsCloseInvalidMessage = 4002 // a frame violates the endpoint protocol + bpsCloseBrokerGone = 4003 // the p2p stream to the broker ended + bpsCloseEndOfStream = 4004 // the principal ended the channel with an EOS + bpsCloseAuthTimeout = 4005 // the principal's AUTH did not arrive in time + + bpsMaxCloseReason = 123 // websocket control frame payload limit minus the code + bpsFrameHeader = 1 + 8 // publish frame header: kind | index (big-endian) + bpsMaxFrameSize = bpsFrameHeader + swarm.SocMaxChunkSize + // delivery frame header: kind | owner | challenge | index (big-endian) + bpsDeliveryHeader = 1 + common.AddressLength + bps.ChallengeSize + 8 + + // bpsGranted is the text frame that tells a stream a roster naming its + // owner has reached it: it may now send its AUTH. + bpsGranted = "granted" +) + +// bpsAuthTimeout bounds how long the principal may take to send its AUTH. It +// is counted from before the join and is shorter than the broker's auth +// deadline, so that the broker never blocklists this node for a slow dApp. +var bpsAuthTimeout = 25 * time.Second + +// errBPSInvalidRoster is a roster query parameter that does not parse. +var errBPSInvalidRoster = errors.New("invalid roster") + +// The reasons a socket ends that are the bridge's own; bpsCloseCode maps them, +// and the session's, to close codes. +var ( + errBPSInvalidMessage = errors.New("invalid message") + errBPSAuthTimeout = errors.New("auth timeout") + errBPSBrokerGone = errors.New("broker stream closed") + errBPSClientGone = errors.New("client gone") + errBPSShutdown = errors.New("node shutting down") +) + +type bpsRequest struct { + principal common.Address + topic []byte + broker swarm.Address + owner []byte // the owner of the stream's frames; none, the stream can only listen + position bps.Position // the place on the channel: cursor, roster, rosterCursor +} + +type bpsChallengeMsg struct { + Challenge string `json:"challenge"` + Broker string `json:"broker"` + DataTopic string `json:"dataTopic"` + AuthTopic string `json:"authTopic"` + RosterTopic string `json:"rosterTopic,omitempty"` // the principal's only + EOSTopic string `json:"eosTopic,omitempty"` // the principal's only +} + +// newBPSChallengeMsg returns what a dApp needs to sign under the stream's +// challenge: the session topics of the kinds its owner may publish. +func newBPSChallengeMsg(req bpsRequest, challenge []byte, isPrincipal bool) (bpsChallengeMsg, error) { + msg := bpsChallengeMsg{ + Challenge: hex.EncodeToString(challenge), + Broker: req.broker.String(), + } + type sessionTopic struct { + kind bps.Kind + dst *string + } + topics := []sessionTopic{{bps.KindData, &msg.DataTopic}, {bps.KindAuth, &msg.AuthTopic}} + if isPrincipal { + topics = append(topics, sessionTopic{bps.KindRoster, &msg.RosterTopic}, sessionTopic{bps.KindEOS, &msg.EOSTopic}) + } + for _, t := range topics { + topic, err := bps.SessionTopic(t.kind, req.topic, challenge) + if err != nil { + return bpsChallengeMsg{}, fmt.Errorf("session topic: %w", err) + } + *t.dst = hex.EncodeToString(topic) + } + return msg, nil +} + +// bpsDeliveryFrame encodes a delivery for the dApp: kind | owner | challenge | +// index | soc — what it needs to re-verify the chunk, whoever published it. +func bpsDeliveryFrame(msg bps.Message) []byte { + data := make([]byte, 0, bpsDeliveryHeader+len(msg.SOC)) + data = append(data, byte(msg.Kind)) + data = append(data, msg.Owner...) + data = append(data, msg.Challenge...) + data = binary.BigEndian.AppendUint64(data, msg.Index) + return append(data, msg.SOC...) +} + +func (s *Service) bpsWsHandler(w http.ResponseWriter, r *http.Request) { + logger := s.logger.WithName("bps").Build() + req, ok := s.bpsParseRequest(w, r, logger) + if !ok { + return + } + conn, ok := s.bpsUpgrade(w, r, logger) + if !ok { + return + } + s.wsWg.Add(1) + go s.bpsRun(conn, req, logger) +} + +// bpsParseRequest validates the request before the websocket upgrade. +// It writes the HTTP error response itself and reports whether to continue. +func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger log.Logger) (bpsRequest, bool) { + if s.bps == nil { + jsonhttp.ServiceUnavailable(w, "bps unavailable") + return bpsRequest{}, false + } + + paths := struct { + Principal common.Address `map:"principal" validate:"required"` + Topic []byte `map:"topic" validate:"required,len=32"` + }{} + if response := s.mapStructure(mux.Vars(r), &paths); response != nil { + response("invalid path params", logger, w) + return bpsRequest{}, false + } + + queries := struct { + Broker swarm.Address `map:"broker"` + Owner []byte `map:"owner" validate:"omitempty,len=20"` + Cursor uint64 `map:"cursor"` + Roster string `map:"roster"` + RosterCursor uint64 `map:"rosterCursor"` + }{} + if response := s.mapStructure(r.URL.Query(), &queries); response != nil { + response("invalid query params", logger, w) + return bpsRequest{}, false + } + if queries.Broker.IsZero() { + jsonhttp.BadRequest(w, "missing broker") + return bpsRequest{}, false + } + if len(queries.Broker.Bytes()) != swarm.HashSize { + jsonhttp.BadRequest(w, "invalid broker") + return bpsRequest{}, false + } + if s.overlay != nil && queries.Broker.Equal(*s.overlay) { + jsonhttp.BadRequest(w, "broker cannot be this node") + return bpsRequest{}, false + } + + pos, err := bpsPosition(paths.Principal.Bytes(), queries.Cursor, queries.Roster, queries.RosterCursor) + if err != nil { + jsonhttp.BadRequest(w, err.Error()) + return bpsRequest{}, false + } + + return bpsRequest{ + principal: paths.Principal, + topic: paths.Topic, + broker: queries.Broker, + owner: queries.Owner, + position: pos, + }, true +} + +// bpsPosition builds the place on the channel from the query: the principal's +// cursor, and the roster as address:cursor pairs — the grantees it names, each +// with the lowest index of its updates to deliver — at the roster cursor, the +// lowest roster index to accept. A dApp keeps them from the deliveries it +// received and passes them back when it reconnects. +func bpsPosition(principal []byte, cursor uint64, roster string, rosterCursor uint64) (bps.Position, error) { + pos := bps.Position{ + Cursors: map[string]uint64{string(principal): cursor}, + RosterCursor: rosterCursor, + } + if roster == "" { + return pos, nil + } + entries := strings.Split(roster, ",") + if len(entries) > bps.MaxRosterSize { + return pos, fmt.Errorf("%w: more than %d addresses", errBPSInvalidRoster, bps.MaxRosterSize) + } + for _, e := range entries { + addr, index, ok := strings.Cut(e, ":") + if !ok { + return pos, fmt.Errorf("%w: %q is not address:index", errBPSInvalidRoster, e) + } + a, err := hex.DecodeString(strings.TrimPrefix(addr, "0x")) + if err != nil || len(a) != common.AddressLength { + return pos, fmt.Errorf("%w: address %q", errBPSInvalidRoster, addr) + } + if bytes.Equal(a, principal) { + return pos, fmt.Errorf("%w: the principal must not be on the roster", errBPSInvalidRoster) + } + n, err := strconv.ParseUint(index, 10, 64) + if err != nil { + return pos, fmt.Errorf("%w: index %q", errBPSInvalidRoster, index) + } + if _, dup := pos.Cursors[string(a)]; dup { + return pos, fmt.Errorf("%w: address %q twice", errBPSInvalidRoster, addr) + } + pos.Cursors[string(a)] = n + pos.Roster = append(pos.Roster, a) + } + return pos, nil +} + +func (s *Service) bpsUpgrade(w http.ResponseWriter, r *http.Request, logger log.Logger) (*websocket.Conn, bool) { + upgrader := websocket.Upgrader{ + ReadBufferSize: swarm.ChunkSize, + WriteBufferSize: swarm.ChunkSize, + CheckOrigin: s.checkOrigin, + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + e := "upgrade failed" + logger.Debug(e, "error", err) + logger.Error(nil, e) + jsonhttp.InternalServerError(w, e) + return nil, false + } + conn.SetReadLimit(bpsMaxFrameSize) + return conn, true +} + +func (s *Service) bpsWrite(conn *websocket.Conn, messageType int, data []byte) error { + if err := conn.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil { + return err + } + return conn.WriteMessage(messageType, data) +} + +func (s *Service) bpsWriteJSON(conn *websocket.Conn, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + return s.bpsWrite(conn, websocket.TextMessage, b) +} + +// bpsCloseCode maps the reason a socket ends to its close code. +func bpsCloseCode(err error) int { + switch { + case errors.Is(err, bps.ErrEndOfStream): + return bpsCloseEndOfStream + case errors.Is(err, bps.ErrInvalidSOC): + return bpsCloseInvalidSOC + case errors.Is(err, errBPSInvalidMessage), + errors.Is(err, bps.ErrInvalidMessage), + errors.Is(err, bps.ErrInvalidRoster), + errors.Is(err, bps.ErrNotPublisher): + return bpsCloseInvalidMessage + case errors.Is(err, errBPSAuthTimeout): + return bpsCloseAuthTimeout + case errors.Is(err, errBPSShutdown): + return websocket.CloseGoingAway + default: + return bpsCloseBrokerGone + } +} + +// bpsClose sends a close frame with the code and the text of err. It may run +// while the inbox writes. Errors are ignored since the connection is torn down +// right after. +func bpsClose(conn *websocket.Conn, err error) { + reason := err.Error() + if len(reason) > bpsMaxCloseReason { + reason = strings.ToValidUTF8(reason[:bpsMaxCloseReason], "") + } + _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(bpsCloseCode(err), reason), time.Now().Add(writeDeadline)) +} + +// bpsParseFrame splits a frame the dApp sent into kind | index | soc. Whether +// the stream may send it, and whether the chunk is valid, is the session's to say. +func bpsParseFrame(typ int, data []byte) (bps.Kind, uint64, []byte, error) { + if typ != websocket.BinaryMessage { + return 0, 0, nil, fmt.Errorf("%w: expected a binary frame", errBPSInvalidMessage) + } + if len(data) < bpsFrameHeader { + return 0, 0, nil, fmt.Errorf("%w: short frame", errBPSInvalidMessage) + } + return bps.Kind(data[0]), binary.BigEndian.Uint64(data[1:bpsFrameHeader]), data[bpsFrameHeader:], nil +} + +// bpsRun bridges a dApp's socket and its stream to the broker. The owner settles +// what the stream is, as it does on the wire: the principal's publishes once it +// has sent its AUTH, which is due within bpsAuthTimeout; any other listens, for +// as long as it likes, and may send its AUTH once told it is granted; with none +// the stream can only listen, and is sent no challenge. The outbox and the inbox +// run until either ends, or the node shuts down; the first reason to stop decides +// how the socket closes. +func (s *Service) bpsRun(conn *websocket.Conn, req bpsRequest, logger log.Logger) { + defer s.wsWg.Done() + defer func() { _ = conn.Close() }() + + ctx, stop := context.WithCancelCause(context.Background()) + defer stop(nil) + go func() { + select { + case <-s.quit: + stop(errBPSShutdown) + case <-ctx.Done(): + } + }() + + isPrincipal := bytes.Equal(req.owner, req.principal.Bytes()) + authTimer := time.NewTimer(bpsAuthTimeout) + defer authTimer.Stop() + if !isPrincipal { + authTimer.Stop() + } + + sess, err := s.bps.Join(ctx, bps.JoinRequest{ + Broker: req.broker, + Spec: bps.CohortSpec{Topic: req.topic, Principal: req.principal.Bytes()}, + Owner: req.owner, + Position: req.position, + }) + if err != nil { + logger.Debug("join failed", "broker", req.broker, "error", err) + stop(fmt.Errorf("%w: %w", errBPSBrokerGone, err)) + bpsClose(conn, context.Cause(ctx)) + return + } + defer sess.Close() + + // a stream with no owner signs nothing: it has no use for the challenge + if len(req.owner) > 0 { + msg, err := newBPSChallengeMsg(req, sess.Challenge(), isPrincipal) + if err != nil { + logger.Debug("challenge message failed", "error", err) + bpsClose(conn, fmt.Errorf("%w: %w", errBPSBrokerGone, err)) + return + } + if err := s.bpsWriteJSON(conn, msg); err != nil { + logger.Debug("ws write failed", "error", err) + return + } + } + + var wg sync.WaitGroup + wg.Go(func() { stop(bpsHandleOutbox(ctx, conn, sess, authTimer)) }) + wg.Go(func() { stop(s.bpsHandleInbox(ctx, conn, sess, isPrincipal, authTimer.C, logger)) }) + + <-ctx.Done() + if err := context.Cause(ctx); !errors.Is(err, errBPSClientGone) { + bpsClose(conn, err) + } + // ends the outbox's read + _ = conn.Close() + wg.Wait() +} + +// bpsHandleOutbox takes what the dApp sends out through the session, until the +// socket fails or the session refuses a frame, and returns why it stopped. It +// runs even while the stream may send nothing: only reading sees a client +// that has gone, and a frame sent too early is refused by the session. +func bpsHandleOutbox(ctx context.Context, conn *websocket.Conn, sess bps.Session, authTimer *time.Timer) error { + for { + typ, data, err := conn.ReadMessage() + if err != nil { + return errBPSClientGone + } + kind, index, chunk, err := bpsParseFrame(typ, data) + if err != nil { + return err + } + if err := sess.Publish(ctx, kind, index, chunk); err != nil { + return err + } + if kind == bps.KindAuth { + authTimer.Stop() + } + } +} + +// bpsHandleInbox brings what the stream receives in to the dApp, until the +// stream or the socket ends or the principal's AUTH is overdue, and returns why +// it stopped. It is the only writer of frames to the socket but the closing one. +func (s *Service) bpsHandleInbox(ctx context.Context, conn *websocket.Conn, sess bps.Session, isPrincipal bool, authC <-chan time.Time, logger log.Logger) error { + ticker := time.NewTicker(s.WsPingPeriod) + defer ticker.Stop() + + // the principal needs no cue; any other stream is told of its + var granted <-chan struct{} + if !isPrincipal { + granted = sess.Granted() + } + + for { + var err error + select { + case <-ctx.Done(): + return ctx.Err() + case <-sess.Done(): + if err := sess.Err(); err != nil { + return err + } + return errBPSBrokerGone + case <-authC: + return errBPSAuthTimeout + case <-granted: + granted = nil + err = s.bpsWrite(conn, websocket.TextMessage, []byte(bpsGranted)) + case msg := <-sess.Messages(): + err = s.bpsWrite(conn, websocket.BinaryMessage, bpsDeliveryFrame(msg)) + case <-ticker.C: + err = s.bpsWrite(conn, websocket.PingMessage, nil) + } + if err != nil { + logger.Debug("ws write failed", "error", err) + return errBPSClientGone + } + } +} diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go new file mode 100644 index 00000000000..7287bf50157 --- /dev/null +++ b/pkg/api/bps_test.go @@ -0,0 +1,725 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package api_test + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/jsonhttp" + "github.com/ethersphere/bee/v2/pkg/jsonhttp/jsonhttptest" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/bee/v2/pkg/util/testutil" + "github.com/gorilla/websocket" +) + +type fakeBPS struct { + sess *fakeSession + joinErr error + + mu sync.Mutex + req bps.JoinRequest +} + +func (f *fakeBPS) Join(_ context.Context, req bps.JoinRequest) (bps.Session, error) { + f.mu.Lock() + f.req = req + f.mu.Unlock() + if f.joinErr != nil { + return nil, f.joinErr + } + // as the session does: the principal needs no cue + if bytes.Equal(req.Owner, req.Spec.Principal) { + close(f.sess.granted) + } + return f.sess, nil +} + +type published struct { + kind bps.Kind + index uint64 + soc []byte +} + +type fakeSession struct { + challenge []byte + msgs chan bps.Message + granted chan struct{} + done chan struct{} + published chan published + closed <-chan struct{} // closed by Close + cancel context.CancelFunc // Close, as the session's: a request, any number of times + + publishErr error + err error +} + +func newFakeSession() *fakeSession { + challenge := make([]byte, 32) + copy(challenge, "bps-test-challenge") + ctx, cancel := context.WithCancel(context.Background()) + return &fakeSession{ + challenge: challenge, + msgs: make(chan bps.Message), + granted: make(chan struct{}), + done: make(chan struct{}), + published: make(chan published, 8), + closed: ctx.Done(), + cancel: cancel, + } +} + +func (f *fakeSession) Challenge() []byte { return f.challenge } +func (f *fakeSession) Messages() <-chan bps.Message { return f.msgs } +func (f *fakeSession) Position() bps.Position { return bps.Position{} } +func (f *fakeSession) Granted() <-chan struct{} { return f.granted } +func (f *fakeSession) Done() <-chan struct{} { return f.done } +func (f *fakeSession) Err() error { return f.err } + +// Publish refuses, as the session does, what comes before the grant. The rest +// of what the session refuses is told by publishErr. +func (f *fakeSession) Publish(_ context.Context, kind bps.Kind, index uint64, b []byte) error { + select { + case <-f.granted: + default: + return bps.ErrNotPublisher + } + if f.publishErr != nil { + return f.publishErr + } + f.published <- published{kind: kind, index: index, soc: b} + return nil +} + +func (f *fakeSession) Close() error { + f.cancel() + return nil +} + +// end simulates the broker stream ending with err; a test calls it once. +func (f *fakeSession) end(err error) { + f.err = err + close(f.done) +} + +type bpsFixture struct { + client *http.Client + listener string + bps *fakeBPS + sess *fakeSession + signer crypto.Signer + principal []byte + owner []byte + topic []byte + broker swarm.Address + self swarm.Address +} + +func newBPSFixture(t *testing.T) *bpsFixture { + t.Helper() + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + principal, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + topic := make([]byte, 32) + copy(topic, "bps-test-topic") + owner := make([]byte, 20) + copy(owner, "bps-test-subscriber") + sess := newFakeSession() + f := &bpsFixture{ + bps: &fakeBPS{sess: sess}, + sess: sess, + signer: signer, + principal: principal.Bytes(), + owner: owner, + topic: topic, + broker: swarm.RandAddress(t), + self: swarm.RandAddress(t), + } + f.client, _, f.listener, _, _ = newTestServer(t, testServerOptions{ + Bps: f.bps, + Overlay: f.self, + WsPingPeriod: 60 * time.Second, + }) + return f +} + +func (f *bpsFixture) path() string { + return fmt.Sprintf("/bps/feeds/subscribe/%s/%s", hex.EncodeToString(f.principal), hex.EncodeToString(f.topic)) +} + +// dial opens the socket as the fixture's owner, which listens. +func (f *bpsFixture) dial(t *testing.T) *websocket.Conn { + t.Helper() + return f.dialQuery(t, f.owner, "") +} + +// dialPrincipal opens the socket as the principal. +func (f *bpsFixture) dialPrincipal(t *testing.T) *websocket.Conn { + t.Helper() + return f.dialQuery(t, f.principal, "") +} + +// dialQuery opens the socket as owner, if any, with more query parameters. +func (f *bpsFixture) dialQuery(t *testing.T, owner []byte, query string) *websocket.Conn { + t.Helper() + q := "broker=" + f.broker.String() + query + if owner != nil { + q += "&owner=" + hex.EncodeToString(owner) + } + u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path(), RawQuery: q} + conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + t.Fatalf("dial %s: %v", u.String(), err) + } + testutil.CleanupCloser(t, conn) + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + return conn +} + +// signedSOC returns the SOC bytes of the session feed update of kind at index, +// signed by signer under challenge. +func (f *bpsFixture) signedSOC(t *testing.T, signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { + t.Helper() + id, err := bps.ID(kind, f.topic, challenge, index) + if err != nil { + t.Fatal(err) + } + ch, err := cac.New(payload) + if err != nil { + t.Fatal(err) + } + c, err := soc.New(id, ch).Sign(signer) + if err != nil { + t.Fatal(err) + } + return c.Data() +} + +// authenticate sends the session's AUTH and waits for the bridge to forward it. +func authenticate(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() + auth := f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindAuth, 0, auth) +} + +// frame returns a publish frame kind | index | soc. +func frame(kind bps.Kind, index uint64, chunk []byte) []byte { + b := []byte{byte(kind)} + b = binary.BigEndian.AppendUint64(b, index) + return append(b, chunk...) +} + +func expectClose(t *testing.T, conn *websocket.Conn, code int) { + t.Helper() + for { + _, _, err := conn.ReadMessage() + if err == nil { + continue + } + var ce *websocket.CloseError + if !errors.As(err, &ce) { + t.Fatalf("want close error with code %d, got %v", code, err) + } + if ce.Code != code { + t.Fatalf("want close code %d, got %d (%q)", code, ce.Code, ce.Text) + } + return + } +} + +// deliver hands the bridge a delivery, as the session does. +func deliver(t *testing.T, f *bpsFixture, msg bps.Message) { + t.Helper() + select { + case f.sess.msgs <- msg: + case <-time.After(5 * time.Second): + t.Fatal("the bridge takes no delivery") + } +} + +// expectOpen checks that the socket stays open for d: a delivery still arrives. +func expectOpen(t *testing.T, f *bpsFixture, conn *websocket.Conn, d time.Duration) { + t.Helper() + time.Sleep(d) + deliver(t, f, bps.Message{Kind: bps.KindData, Owner: f.principal, SOC: []byte("soc"), Challenge: f.sess.challenge, Index: 1}) + for { + typ, _, err := conn.ReadMessage() + if err != nil { + t.Fatalf("socket closed: %v", err) + } + if typ == websocket.BinaryMessage { + return + } + } +} + +// waitSessionClosed fails the test if the session is not closed in time. +func waitSessionClosed(t *testing.T, f *bpsFixture) { + t.Helper() + select { + case <-f.sess.closed: + case <-time.After(5 * time.Second): + t.Fatal("session not closed") + } +} + +// bpsChallenge is the challenge message as a dApp reads it. +type bpsChallenge struct { + Challenge string `json:"challenge"` + Broker string `json:"broker"` + DataTopic string `json:"dataTopic"` + AuthTopic string `json:"authTopic"` + RosterTopic string `json:"rosterTopic"` + EOSTopic string `json:"eosTopic"` +} + +// readChallenge reads the challenge frame and checks the fields every stream gets. +func readChallenge(t *testing.T, f *bpsFixture, conn *websocket.Conn) bpsChallenge { + t.Helper() + var m bpsChallenge + if err := conn.ReadJSON(&m); err != nil { + t.Fatal(err) + } + dataTopic, _ := bps.SessionTopic(bps.KindData, f.topic, f.sess.challenge) + authTopic, _ := bps.SessionTopic(bps.KindAuth, f.topic, f.sess.challenge) + if m.Broker != f.broker.String() || + m.DataTopic != hex.EncodeToString(dataTopic) || m.AuthTopic != hex.EncodeToString(authTopic) { + t.Fatalf("unexpected challenge message %+v", m) + } + if m.Challenge != hex.EncodeToString(f.sess.challenge) { + t.Fatalf("challenge %q, want %x", m.Challenge, f.sess.challenge) + } + return m +} + +func expectPublished(t *testing.T, f *bpsFixture, kind bps.Kind, index uint64, chunk []byte) { + t.Helper() + select { + case got := <-f.sess.published: + if got.kind != kind || got.index != index || !bytes.Equal(got.soc, chunk) { + t.Fatalf("published %d/%d %x, want %d/%d %x", got.kind, got.index, got.soc, kind, index, chunk) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for publish") + } +} + +func joinRequest(f *bpsFixture) bps.JoinRequest { + f.bps.mu.Lock() + defer f.bps.mu.Unlock() + return f.bps.req +} + +func TestBPSPreUpgrade(t *testing.T) { + t.Parallel() + + f := newBPSFixture(t) + nilClient, _, _, _, _ := newTestServer(t, testServerOptions{}) + principal := hex.EncodeToString(f.principal) + topic := hex.EncodeToString(f.topic) + owner := "&owner=" + hex.EncodeToString(f.owner) + base := fmt.Sprintf("/bps/feeds/subscribe/%s/%s", principal, topic) + + jsonhttptest.Request(t, f.client, http.MethodGet, base, http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "missing broker", + }), + ) + jsonhttptest.Request(t, f.client, http.MethodGet, base+"?broker=zz"+owner, http.StatusBadRequest) + jsonhttptest.Request(t, f.client, http.MethodGet, + fmt.Sprintf("/bps/feeds/subscribe/%s/%s?broker=%s%s", "zz", topic, f.broker, owner), http.StatusBadRequest) + jsonhttptest.Request(t, f.client, http.MethodGet, + fmt.Sprintf("/bps/feeds/subscribe/%s/%s?broker=%s%s", principal, "abcd", f.broker, owner), http.StatusBadRequest) + jsonhttptest.Request(t, f.client, http.MethodGet, base+"?broker=abcd"+owner, http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "invalid broker", + }), + ) + jsonhttptest.Request(t, f.client, http.MethodGet, base+"?broker="+f.self.String()+owner, http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "broker cannot be this node", + }), + ) + jsonhttptest.Request(t, f.client, http.MethodGet, base+"?broker="+f.broker.String()+"&owner=abcd", http.StatusBadRequest) + jsonhttptest.Request(t, nilClient, http.MethodGet, base+"?broker="+f.broker.String()+owner, http.StatusServiceUnavailable) +} + +// TestBPSListener: a stream declaring no address, or any but the principal's, listens. +func TestBPSListener(t *testing.T) { + t.Parallel() + + t.Run("forwards deliveries", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t) + if m := readChallenge(t, f, conn); m.RosterTopic != "" || m.EOSTopic != "" { + t.Fatalf("the service topics are the principal's: %+v", m) + } + chunk := f.signedSOC(t, f.signer, bps.KindData, f.sess.challenge, 7, []byte("hello cohort")) + deliver(t, f, bps.Message{Kind: bps.KindData, Owner: f.principal, SOC: chunk, Challenge: f.sess.challenge, Index: 7}) + typ, got, err := conn.ReadMessage() + if err != nil { + t.Fatal(err) + } + want := append([]byte{byte(bps.KindData)}, f.principal...) + want = append(want, f.sess.challenge...) + want = append(want, 0, 0, 0, 0, 0, 0, 0, 7) + want = append(want, chunk...) + if typ != websocket.BinaryMessage || !bytes.Equal(got, want) { + t.Fatalf("got type %d %x, want binary %x", typ, got, want) + } + req := joinRequest(f) + if !req.Broker.Equal(f.broker) || !bytes.Equal(req.Spec.Principal, f.principal) || !bytes.Equal(req.Owner, f.owner) || !bytes.Equal(req.Spec.Topic, f.topic) { + t.Fatalf("unexpected join request %+v", req) + } + }) + + t.Run("no owner joins with none, gets no challenge, and may send nothing", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dialQuery(t, nil, "") + deliver(t, f, bps.Message{Kind: bps.KindData, Owner: f.principal, SOC: []byte("soc"), Challenge: f.sess.challenge, Index: 1}) + if typ, _, err := conn.ReadMessage(); err != nil || typ != websocket.BinaryMessage { + t.Fatalf("want the delivery as the first frame, got %d %v", typ, err) + } + if req := joinRequest(f); len(req.Owner) != 0 { + t.Fatalf("joined with owner %x", req.Owner) + } + auth := f.signedSOC(t, f.signer, bps.KindAuth, f.sess.challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectClose(t, conn, api.BPSCloseInvalidMessage) + }) + + t.Run("a frame before the grant closes 4002", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t) + readChallenge(t, f, conn) + auth := f.signedSOC(t, f.signer, bps.KindAuth, f.sess.challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectClose(t, conn, api.BPSCloseInvalidMessage) + waitSessionClosed(t, f) + }) + + t.Run("broker gone closes 4003", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t) + readChallenge(t, f, conn) + f.sess.end(errors.New("stream reset")) + expectClose(t, conn, api.BPSCloseBrokerGone) + waitSessionClosed(t, f) + }) + + t.Run("join error closes 4003", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.bps.joinErr = errors.New("broker unreachable") + conn := f.dial(t) + expectClose(t, conn, api.BPSCloseBrokerGone) + }) + + t.Run("end of stream closes 4004", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dial(t) + readChallenge(t, f, conn) + f.sess.end(bps.ErrEndOfStream) + expectClose(t, conn, api.BPSCloseEndOfStream) + }) +} + +// TestBPSGrant: a listener is told when a roster naming it has reached its +// stream, and may then send its AUTH and publish. +func TestBPSGrant(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + key, _ := crypto.GenerateSecp256k1Key() + seatSigner := crypto.NewDefaultSigner(key) + seatAddr, _ := seatSigner.EthereumAddress() + seat := seatAddr.Bytes() + + conn := f.dialQuery(t, seat, "") + readChallenge(t, f, conn) + if req := joinRequest(f); !bytes.Equal(req.Owner, seat) { + t.Fatalf("unexpected join request %+v", req) + } + + // listening until the grant + deliver(t, f, bps.Message{Kind: bps.KindData, Owner: f.principal, SOC: []byte("soc"), Challenge: f.sess.challenge}) + if typ, _, err := conn.ReadMessage(); err != nil || typ != websocket.BinaryMessage { + t.Fatalf("want a delivery, got %d %v", typ, err) + } + + close(f.sess.granted) + typ, msg, err := conn.ReadMessage() + if err != nil || typ != websocket.TextMessage || string(msg) != api.BPSGranted { + t.Fatalf("want granted, got %d %q %v", typ, msg, err) + } + auth := f.signedSOC(t, seatSigner, bps.KindAuth, f.sess.challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindAuth, 0, auth) + data := f.signedSOC(t, seatSigner, bps.KindData, f.sess.challenge, 0, []byte("from the seat")) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, data)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindData, 0, data) +} + +// TestBPSPrincipal: a stream declaring the principal's address publishes, service +// messages included, and receives what the others publish. +func TestBPSPrincipal(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dialPrincipal(t) + m := readChallenge(t, f, conn) + rosterTopic, _ := bps.SessionTopic(bps.KindRoster, f.topic, f.sess.challenge) + eosTopic, _ := bps.SessionTopic(bps.KindEOS, f.topic, f.sess.challenge) + if m.RosterTopic != hex.EncodeToString(rosterTopic) || m.EOSTopic != hex.EncodeToString(eosTopic) { + t.Fatalf("unexpected service topics %+v", m) + } + if req := joinRequest(f); !bytes.Equal(req.Owner, f.principal) { + t.Fatalf("joined as %x, want the principal %x", req.Owner, f.principal) + } + authenticate(t, f, conn, f.sess.challenge) + + data := f.signedSOC(t, f.signer, bps.KindData, f.sess.challenge, 3, []byte("broadcast")) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 3, data)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindData, 3, data) + + ros := f.signedSOC(t, f.signer, bps.KindRoster, f.sess.challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindRoster, 0, ros)); err != nil { + t.Fatal(err) + } + expectPublished(t, f, bps.KindRoster, 0, ros) + + seat := make([]byte, 20) + copy(seat, "bps-test-seat") + deliver(t, f, bps.Message{Kind: bps.KindData, Owner: seat, SOC: []byte("soc"), Challenge: f.sess.challenge, Index: 1}) + typ, got, err := conn.ReadMessage() + if err != nil { + t.Fatal(err) + } + if typ != websocket.BinaryMessage || got[0] != byte(bps.KindData) || !bytes.Equal(got[1:21], seat) { + t.Fatalf("got type %d %x", typ, got) + } +} + +// TestBPSFrameErrors: what is wrong with a frame as such is the bridge's to say. +func TestBPSFrameErrors(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + send func(conn *websocket.Conn) error + }{ + {"short frame", func(conn *websocket.Conn) error { return conn.WriteMessage(websocket.BinaryMessage, []byte{1, 2}) }}, + {"text frame", func(conn *websocket.Conn) error { return conn.WriteJSON(map[string]string{"type": "auth"}) }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dialPrincipal(t) + readChallenge(t, f, conn) + if err := tc.send(conn); err != nil { + t.Fatal(err) + } + expectClose(t, conn, api.BPSCloseInvalidMessage) + waitSessionClosed(t, f) + if len(f.sess.published) != 0 { + t.Fatal("rejected frame reached the session") + } + }) + } +} + +// TestBPSPublishErrors: what the session refuses closes the socket with the code +// its error maps to. +func TestBPSPublishErrors(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + err error + code int + }{ + {fmt.Errorf("id not derived from frame: %w", bps.ErrInvalidSOC), api.BPSCloseInvalidSOC}, + {fmt.Errorf("data before the auth: %w", bps.ErrInvalidMessage), api.BPSCloseInvalidMessage}, + {fmt.Errorf("the admin must not be on the roster: %w", bps.ErrInvalidRoster), api.BPSCloseInvalidMessage}, + {bps.ErrNotPublisher, api.BPSCloseInvalidMessage}, + {errors.New("write failed"), api.BPSCloseBrokerGone}, + } { + t.Run(tc.err.Error(), func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + f.sess.publishErr = tc.err + conn := f.dialPrincipal(t) + readChallenge(t, f, conn) + auth := f.signedSOC(t, f.signer, bps.KindAuth, f.sess.challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { + t.Fatal(err) + } + expectClose(t, conn, tc.code) + waitSessionClosed(t, f) + }) + } +} + +func TestBPSCloseCode(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + err error + code int + }{ + {bps.ErrInvalidSOC, api.BPSCloseInvalidSOC}, + {api.ErrBPSInvalidMessage, api.BPSCloseInvalidMessage}, + {bps.ErrInvalidMessage, api.BPSCloseInvalidMessage}, + {bps.ErrInvalidRoster, api.BPSCloseInvalidMessage}, + {bps.ErrNotPublisher, api.BPSCloseInvalidMessage}, + {api.ErrBPSBrokerGone, api.BPSCloseBrokerGone}, + {errors.New("stream reset"), api.BPSCloseBrokerGone}, + {bps.ErrEndOfStream, api.BPSCloseEndOfStream}, + {fmt.Errorf("wrapped: %w", bps.ErrEndOfStream), api.BPSCloseEndOfStream}, + {api.ErrBPSAuthTimeout, api.BPSCloseAuthTimeout}, + {api.ErrBPSShutdown, websocket.CloseGoingAway}, + } { + if got := api.BPSCloseCode(tc.err); got != tc.code { + t.Errorf("%v: got %d, want %d", tc.err, got, tc.code) + } + } +} + +func TestBPSClientGone(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dialPrincipal(t) + readChallenge(t, f, conn) + // Send a close frame rather than Close, which the dial cleanup would + // report as an error on the already closed connection. + _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + waitSessionClosed(t, f) +} + +func TestBPSOversizedFrame(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dialPrincipal(t) + readChallenge(t, f, conn) + _ = conn.WriteMessage(websocket.BinaryMessage, make([]byte, 5000)) + expectClose(t, conn, websocket.CloseMessageTooBig) + waitSessionClosed(t, f) +} + +// Not parallel: mutates the package-level auth timeout. +func TestBPSAuthTimeout(t *testing.T) { + const timeout = 100 * time.Millisecond + restore := api.SetBPSAuthTimeout(timeout) + defer restore() + + t.Run("the principal's AUTH overdue closes 4005", func(t *testing.T) { + f := newBPSFixture(t) + conn := f.dialPrincipal(t) + readChallenge(t, f, conn) + expectClose(t, conn, api.BPSCloseAuthTimeout) + waitSessionClosed(t, f) + }) + + t.Run("the principal's AUTH in time stops the deadline", func(t *testing.T) { + f := newBPSFixture(t) + conn := f.dialPrincipal(t) + readChallenge(t, f, conn) + authenticate(t, f, conn, f.sess.challenge) + expectOpen(t, f, conn, 2*timeout) + }) + + t.Run("a listener has no deadline, granted or not", func(t *testing.T) { + f := newBPSFixture(t) + conn := f.dial(t) + readChallenge(t, f, conn) + expectOpen(t, f, conn, 2*timeout) + close(f.sess.granted) + expectOpen(t, f, conn, 2*timeout) + }) +} + +func TestBPSPosition(t *testing.T) { + t.Parallel() + seat1 := bytes.Repeat([]byte{0x11}, 20) + seat2 := bytes.Repeat([]byte{0x22}, 20) + + t.Run("cursor, roster and rosterCursor reach the join", func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + conn := f.dialQuery(t, f.owner, "&cursor=7&rosterCursor=3&roster="+hex.EncodeToString(seat1)+":4,0x"+hex.EncodeToString(seat2)+":0") + readChallenge(t, f, conn) + pos := joinRequest(f).Position + if pos.RosterCursor != 3 || len(pos.Roster) != 2 || !bytes.Equal(pos.Roster[0], seat1) || !bytes.Equal(pos.Roster[1], seat2) { + t.Fatalf("roster %x at %d", pos.Roster, pos.RosterCursor) + } + if pos.Cursors[string(f.principal)] != 7 || pos.Cursors[string(seat1)] != 4 || pos.Cursors[string(seat2)] != 0 || len(pos.Cursors) != 3 { + t.Fatalf("cursors %v", pos.Cursors) + } + }) + + for _, tc := range []struct { + name, roster string + }{ + {"no index", hex.EncodeToString(seat1)}, + {"short address", "1122:4"}, + {"bad index", hex.EncodeToString(seat1) + ":x"}, + {"the principal", "PRINCIPAL:1"}, + {"twice", hex.EncodeToString(seat1) + ":1," + hex.EncodeToString(seat1) + ":2"}, + {"too many", strings.Repeat(hex.EncodeToString(seat1)+":1,", bps.MaxRosterSize) + hex.EncodeToString(seat2) + ":1"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := newBPSFixture(t) + roster := strings.ReplaceAll(tc.roster, "PRINCIPAL", hex.EncodeToString(f.principal)) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, f.path()+"?broker="+f.broker.String()+"&owner="+hex.EncodeToString(f.owner)+"&roster="+roster, nil) + if err != nil { + t.Fatal(err) + } + resp, err := f.client.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status %d, want 400", resp.StatusCode) + } + }) + } +} diff --git a/pkg/api/export_test.go b/pkg/api/export_test.go index c96b9e81ae8..9f33771fe05 100644 --- a/pkg/api/export_test.go +++ b/pkg/api/export_test.go @@ -5,6 +5,8 @@ package api import ( + "time" + "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/gorilla/mux" @@ -162,3 +164,28 @@ func NewParseError(entry, value string, cause error) error { const RootPath = rootPath func (s *Service) Router() *mux.Router { return s.router } + +const ( + BPSCloseInvalidSOC = bpsCloseInvalidSOC + BPSCloseInvalidMessage = bpsCloseInvalidMessage + BPSCloseBrokerGone = bpsCloseBrokerGone + BPSCloseEndOfStream = bpsCloseEndOfStream + BPSCloseAuthTimeout = bpsCloseAuthTimeout + BPSGranted = bpsGranted +) + +var ( + ErrBPSInvalidMessage = errBPSInvalidMessage + ErrBPSAuthTimeout = errBPSAuthTimeout + ErrBPSBrokerGone = errBPSBrokerGone + ErrBPSShutdown = errBPSShutdown + BPSCloseCode = bpsCloseCode +) + +// SetBPSAuthTimeout overrides the auth timeout and returns a restore func. +// Callers must not run in parallel with other bps tests. +func SetBPSAuthTimeout(d time.Duration) func() { + prev := bpsAuthTimeout + bpsAuthTimeout = d + return func() { bpsAuthTimeout = prev } +} diff --git a/pkg/api/router.go b/pkg/api/router.go index 7c11fd04b58..2fa4210db97 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -392,6 +392,10 @@ func (s *Service) mountAPI() { ), }) + handle("/bps/feeds/subscribe/{principal}/{topic}", jsonhttp.MethodHandler{ + http.MethodGet: http.HandlerFunc(s.bpsWsHandler), + }) + handle("/tags", jsonhttp.MethodHandler{ http.MethodGet: http.HandlerFunc(s.listTagsHandler), http.MethodPost: web.ChainHandlers( diff --git a/pkg/api/status_test.go b/pkg/api/status_test.go index 5844e86547e..440dcf2022c 100644 --- a/pkg/api/status_test.go +++ b/pkg/api/status_test.go @@ -74,7 +74,6 @@ func TestGetStatus(t *testing.T) { jsonhttptest.WithExpectedJSONResponse(ssr), ) }) - } // TestGetStatusPeersIncludesBootnodes is a regression test for diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go new file mode 100644 index 00000000000..4a36d5e8a3b --- /dev/null +++ b/pkg/bps/bps.go @@ -0,0 +1,1483 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bps implements BPS-lite (SWIP-74) with the explicit multi-publisher +// mode of SWIP-87: a live stream over feeds, through one broker, one hop. +// +// A cohort is identified by its spec {topic, FEED_TOPIC, principal}. Every +// peer joins with the spec and, unless it will only listen, the stream's owner — +// the wallet address signing its feed update SOCs — and is answered with a random +// per-stream challenge. A stream declaring the principal — the admin — is +// pending until its AUTH — the empty chunk at index 0 of the session's AUTH feed +// — authenticates it; any other stream listens, and its AUTH upgrades it once a +// roster naming its owner has been delivered on it. A publisher stream sends +// ordinary single-owner chunks that are updates of its session feed on +// keccak256(topic | challenge), and the admin's publisher stream also its service +// messages: ROSTER, the publisher set, and EOS, the end of the channel. The +// broker keeps a cursor per publisher and per service kind, and delivers every +// accepted frame to every receiving stream not bound to its author. +package bps + +import ( + "bytes" + "context" + "crypto/rand" + "errors" + "fmt" + "maps" + "slices" + "sync" + "time" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// loggerName is the tree path name of the logger for this package. +const loggerName = "bps" + +const ( + protocolName = "bps" + protocolVersion = "1.0.0" + streamName = "bps" +) + +var ( + errNotBroker = errors.New("bps: not a broker") + errJoinRefused = errors.New("bps: join refused") + errInvalidJoin = errors.New("bps: invalid join") + errSessionClosed = errors.New("bps: session closed") + errProtocolError = errors.New("bps: protocol error") + errViolation = errors.New("bps: protocol violation") + errWrongStream = errors.New("bps: frame not what the stream may send") + errCohortReclaimed = errors.New("bps: cohort reclaimed") + errFull = errors.New("bps: capacity bound") + errInvalidDelivery = errors.New("bps: invalid delivery") + errChannelEnded = errors.New("bps: channel ended") + + // ErrEndOfStream is a session's Err after the admin's EOS ended the channel. + ErrEndOfStream = errors.New("bps: end of stream") + // ErrNotPublisher refuses a publication the session may not make: not + // granted, revoked, or a service message from anyone but the admin. + ErrNotPublisher = errors.New("bps: session is not a publisher") +) + +// ForgingBrokerBlocklistDuration is how long a subscriber avoids a broker that +// forwarded a delivery it should have refused — one that does not validate, or +// whose owner the roster does not admit — which no honest broker sends. +const ForgingBrokerBlocklistDuration = 10 * 24 * time.Hour + +// Options are the broker's resource bounds. Zero values take the defaults. +type Options struct { + MaxSubsPerCohort int // subscriber streams per cohort + MaxBrokerCohorts int // live cohorts per broker + MaxCohortsPerConnection int // cohorts per peer connection + MaxPeerStreamsPerCohort int // streams per peer connection per cohort + MaxPublisherStreams int // publisher streams per address per cohort + QueueCap int // outbound frames per receiving stream + // + InactiveTimeout time.Duration // reclaims a cohort with no accepted frame + AuthWaitTimeout time.Duration // disconnects a pending stream that has not authenticated + FlushTimeout time.Duration // bounds writing the queue of a stream being closed + // + ProtoBreachBlocklistDuration time.Duration // blocklist duration for a protocol violation + InvalidAuthBlocklistDuration time.Duration // blocklist duration for an auth timeout +} + +// DefaultOptions are the SWIP-74 and SWIP-87 recommended bounds. +var DefaultOptions = Options{ + MaxSubsPerCohort: 1024, + MaxBrokerCohorts: 512, + MaxCohortsPerConnection: 16, + MaxPeerStreamsPerCohort: 2, + MaxPublisherStreams: 2, + InactiveTimeout: 10 * time.Minute, + AuthWaitTimeout: 30 * time.Second, + FlushTimeout: 5 * time.Second, + QueueCap: 64, + ProtoBreachBlocklistDuration: 10 * time.Minute, + InvalidAuthBlocklistDuration: time.Minute, +} + +func (o Options) withDefaults() Options { + d := DefaultOptions + if o.MaxSubsPerCohort > 0 { + d.MaxSubsPerCohort = o.MaxSubsPerCohort + } + if o.MaxBrokerCohorts > 0 { + d.MaxBrokerCohorts = o.MaxBrokerCohorts + } + if o.MaxCohortsPerConnection > 0 { + d.MaxCohortsPerConnection = o.MaxCohortsPerConnection + } + if o.MaxPeerStreamsPerCohort > 0 { + d.MaxPeerStreamsPerCohort = o.MaxPeerStreamsPerCohort + } + if o.MaxPublisherStreams > 0 { + d.MaxPublisherStreams = o.MaxPublisherStreams + } + if o.InactiveTimeout > 0 { + d.InactiveTimeout = o.InactiveTimeout + } + if o.AuthWaitTimeout > 0 { + d.AuthWaitTimeout = o.AuthWaitTimeout + } + if o.FlushTimeout > 0 { + d.FlushTimeout = o.FlushTimeout + } + if o.QueueCap > 0 { + d.QueueCap = o.QueueCap + } + if o.ProtoBreachBlocklistDuration > 0 { + d.ProtoBreachBlocklistDuration = o.ProtoBreachBlocklistDuration + } + if o.InvalidAuthBlocklistDuration > 0 { + d.InvalidAuthBlocklistDuration = o.InvalidAuthBlocklistDuration + } + return d +} + +// Service is the bps protocol service. When broker is set it also serves +// cohorts to inbound joins; otherwise it only joins remote brokers. +type Service struct { + broker bool + streamer p2p.Streamer + blocklister p2p.Blocklister + opts Options + metrics metrics + logger log.Logger + + quit chan struct{} // closed by Close + + mtx sync.Mutex + cohorts map[string]*cohort // by canonical spec + peers map[string]map[string]int // peer overlay -> cohort key -> streams +} + +// New returns a new bps Service. The blocklister may be nil. +func New(streamer p2p.Streamer, blocklister p2p.Blocklister, broker bool, logger log.Logger, o Options) *Service { + return &Service{ + broker: broker, + streamer: streamer, + blocklister: blocklister, + opts: o.withDefaults(), + metrics: newMetrics(), + quit: make(chan struct{}), + cohorts: make(map[string]*cohort), + peers: make(map[string]map[string]int), + logger: logger.WithName(loggerName).Register(), + } +} + +// Protocol returns the p2p protocol specification for bps. +func (s *Service) Protocol() p2p.ProtocolSpec { + return p2p.ProtocolSpec{ + Name: protocolName, + Version: protocolVersion, + StreamSpecs: []p2p.StreamSpec{ + { + Name: streamName, + Handler: s.handler, + }, + }, + } +} + +// Close refuses every later join, forgets every cohort and asks every stream +// in them to reset. A second call does nothing. +func (s *Service) Close() error { + s.mtx.Lock() + defer s.mtx.Unlock() + select { + case <-s.quit: + return nil + default: + } + close(s.quit) + for _, co := range s.cohorts { + co.mtx.Lock() + s.removeCohort(co) + for _, m := range co.members() { + m.requestReset() + } + co.mtx.Unlock() + } + return nil +} + +// Interface is the bps surface consumed by the API. +type Interface interface { + // Join joins a cohort at a broker and returns the live session. + Join(ctx context.Context, req JoinRequest) (Session, error) +} + +// Message is a verified delivery — a DATA update of an admitted publisher, or +// one of the admin's ROSTER and EOS — with the frame fields its id was derived +// from and the owner it validated for. +type Message struct { + Kind Kind + Owner []byte // the publisher; the principal for ROSTER and EOS + SOC []byte + Challenge []byte + Index uint64 + Roster [][]byte // ROSTER only: the publisher set besides the principal +} + +// Position is a subscriber's place on a channel, kept across streams and +// sessions: a cursor per publisher, the latest roster with its cursor, and +// whether the channel has ended. +type Position struct { + Cursors map[string]uint64 // by the publisher's 20-byte address as a string: expect the next DATA feed update at this index or above; 0 = none yet + Roster [][]byte // the latest accepted roster + RosterCursor uint64 // expect the next ROSTER feed update at this index or above; 0 = none yet + Ended bool // an EOS was accepted: the EOS cursor is past 0 +} + +func (p Position) clone() Position { + p.Cursors = maps.Clone(p.Cursors) + if p.Cursors == nil { + p.Cursors = make(map[string]uint64) + } + p.Roster = slices.Clone(p.Roster) + return p +} + +// Session is a single joined p2p stream to a broker. +type Session interface { + // Challenge is the challenge the broker issued for this stream. Every + // chunk published on the session must be signed under it (see ID). + Challenge() []byte + // Messages yields the deliveries that verify end to end against the cohort + // spec, the roster and the session's cursors: DATA from the principal or a + // rostered publisher, and the principal's ROSTER and EOS. It is not closed: + // Done says when the stream has ended, by when every delivery, an EOS last, + // has been handed over. + Messages() <-chan Message + // Position is the session's place on the channel, to pass to a later Join. + Position() Position + // Granted is closed once the session may authenticate: at once for the + // principal, and for any other owner once a roster naming it has been + // delivered on this stream — the cue a grantee waits for before its AUTH, + // which upgrades the stream in place. A publisher whose owner a roster + // drops finds its stream closed after that roster. A stream with no owner is + // never granted: no roster can name it. + Granted() <-chan struct{} + // Publish writes a chunk of the given kind and index, signed under the + // session challenge, to the broker. The broker does not reply, so the + // session refuses what the broker would: ErrNotPublisher unless the owner + // is the principal, or granted and on the latest roster, and for a ROSTER or + // EOS from anyone but the principal; ErrInvalidMessage for anything before + // the AUTH and for a frame no chunk can make right; ErrInvalidSOC for a chunk + // that does not validate; ErrInvalidRoster for a roster it will not send. + Publish(ctx context.Context, kind Kind, index uint64, chunk []byte) error + // Done is closed when the p2p stream has ended. + Done() <-chan struct{} + // Err reports why the stream ended: ErrEndOfStream after an EOS. Valid after + // Done is closed. + Err() error + // Close closes the stream. + Close() error +} + +// CohortSpec names a cohort: the feeds on the topic of the principal and of +// whoever its roster names. The topic binding is always FEED_TOPIC. +type CohortSpec struct { + Topic []byte // 32 bytes: the feed topic + Principal []byte // 20 bytes: the admin +} + +func (c CohortSpec) proto() *pb.CohortSpec { + return &pb.CohortSpec{Topic: c.Topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: c.Principal} +} + +// JoinRequest describes a cohort join. +type JoinRequest struct { + Broker swarm.Address // the broker + Spec CohortSpec // the cohort to join + Owner []byte // 20 bytes or nil: the owner of the stream's frames, the wallet address signing its feed update SOCs; nil, the stream can only listen + Position Position // the subscriber's place on the channel, from an earlier session +} + +// Join joins a cohort at the broker. A peer whose join was refused or whose +// stream was reset must back off before rejoining. +func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { + join := &pb.Join{Cohort: req.Spec.proto(), Owner: req.Owner} + if err := validateJoin(join); err != nil { + return nil, err + } + // a channel the subscriber has seen end is not rejoined + if req.Position.Ended { + return nil, errChannelEnded + } + stream, err := s.streamer.NewStream(ctx, req.Broker, nil, protocolName, protocolVersion, streamName) + if err != nil { + return nil, fmt.Errorf("new stream: %w", err) + } + + w, r := protobuf.NewWriterAndReader(stream) + if err := w.WriteMsgWithContext(ctx, join); err != nil { + _ = stream.Reset() + return nil, fmt.Errorf("write join: %w", err) + } + var ack pb.Ack + if err := r.ReadMsgWithContext(ctx, &ack); err != nil { + _ = stream.Reset() + return nil, fmt.Errorf("read ack: %w", err) + } + if ack.Status != pb.Status_OK { + _ = stream.Reset() + return nil, fmt.Errorf("join %s: %w", ack.Status, errJoinRefused) + } + if len(ack.Challenge) != ChallengeSize { + _ = stream.Reset() + return nil, fmt.Errorf("challenge length %d: %w", len(ack.Challenge), errProtocolError) + } + + sess := newSession(s, req, w, ack.Challenge) + go sess.readLoop(stream, r) + + return sess, nil +} + +// readLoop delivers the broker's frames until the stream ends, and is the one +// place where the session ends: on its way out it alone ends the stream — closed +// after an EOS or a Close, reset on anything else — and then closes Done. Close +// only asks, by cancelling the context, which stops a read or a delivery in +// progress. +func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader) { + defer close(s.done) + defer func() { + if s.err != nil && !errors.Is(s.err, ErrEndOfStream) && s.ctx.Err() == nil { + _ = stream.Reset() + } else { + _ = stream.FullClose() + } + s.cancel() + }() + + for { + var f pb.Broadcast + if s.err = r.ReadMsgWithContext(s.ctx, &f); s.err != nil { + return + } + msg, ok, err := s.accept(&f) + if err != nil { + // a broker that forwards a chunk not valid for the spec, or by an + // owner the roster does not admit, is misbehaving: the stream is ended + s.err = fmt.Errorf("%w: %w", errInvalidDelivery, err) + s.forged(s.err) + return + } + if !ok { + continue + } + select { + case s.rx <- msg: + case <-s.ctx.Done(): + return + } + if msg.Kind == KindEOS { + s.err = ErrEndOfStream + return + } + } +} + +// accept re-verifies a delivery end to end. It returns false for a frame the +// subscriber drops: an AUTH, a kind it does not know, one below its cursor, a +// roster that does not decode. The challenge is not compared: it is the +// publisher stream's, which the subscriber never sees. +func (s *session) accept(f *pb.Broadcast) (Message, bool, error) { + msg := Message{SOC: f.Soc, Challenge: f.Challenge, Index: f.Index} + s.mtx.Lock() + defer s.mtx.Unlock() + switch f.Kind { + case pb.Kind_DATA: + c, err := parseFrame(f, s.spec.Topic) + if err != nil { + return msg, false, err + } + owner := c.OwnerAddress() + if !s.admits(owner) { + return msg, false, fmt.Errorf("owner %x not rostered", owner) + } + k := string(owner) + if f.Index < s.pos.Cursors[k] { + return msg, false, nil + } + s.pos.Cursors[k] = f.Index + 1 + msg.Kind, msg.Owner = KindData, owner + return msg, true, nil + + case pb.Kind_ROSTER: + // the owner of a service chunk is forced to the principal + c, err := verify(f, s.spec.Topic, s.spec.Principal) + if err != nil { + return msg, false, err + } + roster, ok := decodeRosterMsg(payload(c)) + if !ok { + return msg, false, nil + } + if f.Index >= s.pos.RosterCursor { + s.pos.RosterCursor = f.Index + 1 + s.pos.Roster = roster + msg.Kind, msg.Owner, msg.Roster = KindRoster, s.spec.Principal, roster + } + // the cue: a roster naming the owner, delivered on this stream, + // while the latest roster still names it — the broker holds it too + if contains(roster, s.owner) && s.admits(s.owner) { + s.grant() + } + return msg, msg.Kind == KindRoster, nil + + case pb.Kind_EOS: + if _, err := verify(f, s.spec.Topic, s.spec.Principal); err != nil { + return msg, false, err + } + s.pos.Ended = true + msg.Kind, msg.Owner = KindEOS, s.spec.Principal + return msg, true, nil + } + return msg, false, nil +} + +// admits reports whether owner may publish: the principal, or an address on the +// latest roster. Must be called with s.mtx held. +func (s *session) admits(owner []byte) bool { + return bytes.Equal(owner, s.spec.Principal) || contains(s.pos.Roster, owner) +} + +func contains(set [][]byte, a []byte) bool { + for _, b := range set { + if bytes.Equal(a, b) { + return true + } + } + return false +} + +// session is the concrete Session implementation. Its read loop owns its end +// (see readLoop). +type session struct { + svc *Service + broker swarm.Address + ctx context.Context + cancel context.CancelFunc // asks the read loop to stop + wmtx sync.Mutex + w protobuf.Writer + challenge []byte + spec CohortSpec + owner []byte + mtx sync.Mutex + pos Position + authed bool // the AUTH was written: anything else may follow + granted chan struct{} // closed at the cue, by the read loop; at once for the principal + rx chan Message // unbuffered: a delivery is handed over before Done can close + done chan struct{} // closed by the read loop as it returns + err error // set by the read loop before done is closed +} + +// newSession returns the session of a stream joined at the broker; the +// principal's may authenticate at once. +func newSession(svc *Service, req JoinRequest, w protobuf.Writer, challenge []byte) *session { + ctx, cancel := context.WithCancel(context.Background()) + s := &session{ + svc: svc, + broker: req.Broker, + ctx: ctx, + cancel: cancel, + w: w, + challenge: challenge, + spec: req.Spec, + owner: req.Owner, + pos: req.Position.clone(), + granted: make(chan struct{}), + rx: make(chan Message), + done: make(chan struct{}), + } + if bytes.Equal(req.Owner, req.Spec.Principal) { + close(s.granted) + } + return s +} + +func (s *session) Challenge() []byte { return s.challenge } +func (s *session) Messages() <-chan Message { return s.rx } +func (s *session) Granted() <-chan struct{} { return s.granted } +func (s *session) Done() <-chan struct{} { return s.done } +func (s *session) Err() error { return s.err } + +// Close asks the read loop to end the session. It never blocks, and may be +// called any number of times, from any goroutine. +func (s *session) Close() error { + s.cancel() + return nil +} + +func (s *session) Position() Position { + s.mtx.Lock() + defer s.mtx.Unlock() + return s.pos.clone() +} + +// grant closes granted, the cue to authenticate, unless it is closed already. +// Only the read loop calls it, so that the check and the close cannot race. +func (s *session) grant() { + select { + case <-s.granted: + default: + close(s.granted) + } +} + +// forged avoids, for long, a broker that forwarded a delivery it should have +// refused. +func (s *session) forged(err error) { + if s.svc.blocklister == nil { + return + } + if berr := s.svc.blocklister.Blocklist(s.broker, ForgingBrokerBlocklistDuration, err.Error()); berr != nil { + s.svc.logger.Debug("blocklist broker failed", "broker", s.broker, "error", berr) + } +} + +// Publish checks a frame the way the broker will, so that a mistake is told to +// the caller rather than punished at the broker, and writes it. +func (s *session) Publish(ctx context.Context, kind Kind, index uint64, chunk []byte) error { + f := &pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: s.challenge, Index: index} + if err := s.check(f); err != nil { + return err + } + select { + case <-s.done: + return errSessionClosed + default: + } + s.wmtx.Lock() + defer s.wmtx.Unlock() + if err := s.w.WriteMsgWithContext(ctx, f); err != nil { + return fmt.Errorf("write broadcast: %w", err) + } + if f.Kind == pb.Kind_AUTH { + s.mtx.Lock() + s.authed = true + s.mtx.Unlock() + } + // a roster or an EOS the admin publishes is delivered back to it, and it + // admits by it from where it lands in its stream + return nil +} + +// check runs, for a frame about to be published, the broker's checks in the +// broker's order: what the stream may send now, then the frame itself. +func (s *session) check(f *pb.Broadcast) error { + select { + case <-s.granted: + default: + return ErrNotPublisher + } + s.mtx.Lock() + seated, authed := s.admits(s.owner), s.authed + s.mtx.Unlock() + principal := bytes.Equal(s.owner, s.spec.Principal) + if !seated || !principal && isServiceKind(f.Kind) { + return ErrNotPublisher + } + switch f.Kind { + case pb.Kind_DATA, pb.Kind_AUTH, pb.Kind_EOS, pb.Kind_ROSTER: + default: + return fmt.Errorf("unknown kind: %w", ErrInvalidMessage) + } + // a stream may send nothing before its AUTH; the broker would reset it + if !authed && f.Kind != pb.Kind_AUTH { + return fmt.Errorf("%s before the auth: %w", f.Kind, ErrInvalidMessage) + } + // a service kind has passed only from the principal, so the stream's owner is the chunk's + c, err := verify(f, s.spec.Topic, s.owner) + if err != nil { + return err + } + if f.Kind == pb.Kind_ROSTER { + return checkRoster(payload(c), s.spec.Principal) + } + return nil +} + +// checkRoster refuses a roster this node will not send. A receiver drops one +// that does not decode, and an entry equal to the admin's, or repeated, changes +// nothing there; a sender has no reason to send either, and can still be told. +func checkRoster(b, principal []byte) error { + roster, ok := decodeRosterMsg(b) + if !ok { + return ErrInvalidRoster + } + for i, a := range roster { + if bytes.Equal(a, principal) { + return fmt.Errorf("the admin must not be on the roster: %w", ErrInvalidRoster) + } + if contains(roster[:i], a) { + return fmt.Errorf("address %x twice: %w", a, ErrInvalidRoster) + } + } + return nil +} + +// validateJoin rejects a Join whose spec has a value outside SWIP-74 or whose +// owner is neither absent nor 20 bytes. +func validateJoin(join *pb.Join) error { + cohortSpec := join.Cohort + switch { + case cohortSpec == nil: + return fmt.Errorf("no cohort: %w", errInvalidJoin) + case len(cohortSpec.Topic) != swarm.HashSize: + return fmt.Errorf("topic length %d: %w", len(cohortSpec.Topic), errInvalidJoin) + case cohortSpec.Binding != pb.TopicBinding_FEED_TOPIC: + return fmt.Errorf("binding %s: %w", cohortSpec.Binding, errInvalidJoin) + case len(cohortSpec.Principal) != crypto.AddressSize: + return fmt.Errorf("principal length %d: %w", len(cohortSpec.Principal), errInvalidJoin) + case len(join.Owner) != 0 && len(join.Owner) != crypto.AddressSize: + return fmt.Errorf("owner length %d: %w", len(join.Owner), errInvalidJoin) + } + return nil +} + +// handler is the protocol handler on the broker. +func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { + if !s.broker { + _ = stream.Reset() + return errNotBroker + } + w, r := protobuf.NewWriterAndReader(stream) + + var join pb.Join + if err := r.ReadMsgWithContext(ctx, &join); err != nil { + _ = stream.Reset() + return fmt.Errorf("read join: %w", err) + } + if err := validateJoin(&join); err != nil { + return s.refuse(ctx, w, stream, err) + } + // re-marshal so that the cohort key ignores fields BPS-lite does not define + join.Cohort = &pb.CohortSpec{Topic: join.Cohort.Topic, Binding: join.Cohort.Binding, Principal: join.Cohort.Principal} + + co, m, leave, err := s.join(p.Address, &join) + if err != nil { + return s.refuse(ctx, w, stream, err) + } + // detach on every exit, normal or not + defer leave() + + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: m.challenge}); err != nil { + _ = stream.Reset() + return fmt.Errorf("write ack: %w", err) + } + return s.serve(ctx, p.Address, stream, w, r, co, m) +} + +// serve runs the two halves of an attached stream — the reader, handling the +// peer's frames, and the writer, delivering the stream's queue — and is the +// one place where the stream ends: it alone closes or resets it, on its own +// deadlines and on the requests anyone else makes of the member. It returns +// once both halves have stopped. +func (s *Service) serve(ctx context.Context, peer swarm.Address, stream p2p.Stream, w protobuf.Writer, r protobuf.Reader, co *cohort, m *member) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // each half closes its done channel when it stops, the reason left in its + // err; flush is closed here, once: write what is queued, then stop + flush := make(chan struct{}) + readDone, writeDone := make(chan struct{}), make(chan struct{}) + var readErr, writeErr error + go func() { defer close(readDone); readErr = s.read(ctx, r, co, m) }() + go func() { defer close(writeDone); writeErr = write(ctx, w, m.queue, flush) }() + readC := readDone // nil once seen, so that the closed channel is not seen again + + // the auth deadline of a stream admitted pending + var authC <-chan time.Time + if m.authDeadline { + t := time.NewTimer(s.opts.AuthWaitTimeout) + defer t.Stop() + authC = t.C + } + // the flush deadline, from the moment the stream is asked to close + var flushTimer *time.Timer + var flushC <-chan time.Time + defer func() { + if flushTimer != nil { + flushTimer.Stop() + } + }() + + // end closes or resets the stream and waits for both halves to stop + end := func(graceful bool) { + if graceful { + // closing, not resetting, lets the frames already written arrive + _ = stream.Close() + } else { + _ = stream.Reset() + } + cancel() + <-readDone + <-writeDone + } + + for { + // a reset asked for takes precedence over everything else + select { + case <-m.resetReq: + end(false) + return nil + default: + } + select { + case <-m.resetReq: + end(false) + return nil + + case <-m.closeReq: + // an EOS or a revocation: write the queue — the EOS or the reduced + // roster last — then close, within the flush deadline + if flushC == nil { + close(flush) + flushTimer = time.NewTimer(s.opts.FlushTimeout) + flushC = flushTimer.C + } + + case <-flushC: + // the peer did not take its queue in time, the write in progress + // included: the timer is not the writer's, so a stuck write cannot + // hold it off + end(false) + return nil + + case <-writeDone: + // the queue written after a close, or the writer failed + end(flushC != nil && writeErr == nil) + return nil + + case <-readC: + readC = nil + if flushC != nil { + // closing: nothing more is read anyway, the flush decides + continue + } + violation := errors.Is(readErr, errViolation) + if violation && s.blocklister != nil { + if berr := s.blocklister.Blocklist(peer, s.opts.ProtoBreachBlocklistDuration, readErr.Error()); berr != nil { + s.logger.Debug("blocklist failed", "peer_address", peer, "error", berr) + } + } + // a violation, or the peer went away + end(false) + if violation { + return readErr + } + return nil + + case <-authC: + authC = nil + if s.authExpired(peer, co, m) { + end(false) + return nil + } + + case <-ctx.Done(): + end(false) + return nil + } + } +} + +// read handles the peer's frames until the stream ends, and returns why: a +// violation, the peer going away, or ctx ending. +func (s *Service) read(ctx context.Context, r protobuf.Reader, co *cohort, m *member) error { + for { + var f pb.Broadcast + if err := r.ReadMsgWithContext(ctx, &f); err != nil { + return err + } + if err := s.handleFrame(co, m, &f); err != nil { + return err + } + } +} + +// write delivers the stream's queue until ctx ends; once flush is closed, it +// writes what is still queued and returns nil. +func write(ctx context.Context, w protobuf.Writer, queue <-chan *pb.Broadcast, flush <-chan struct{}) error { + for { + select { + case f := <-queue: + if err := w.WriteMsgWithContext(ctx, f); err != nil { + return err + } + case <-flush: + for { + select { + case f := <-queue: + if err := w.WriteMsgWithContext(ctx, f); err != nil { + return err + } + default: + return nil + } + } + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// refuse answers a Join with a non-OK Ack and ends the stream. The status +// follows from the cause — FULL for a capacity bound, REJECTED for anything +// else — so that no caller can answer a refusal with OK. +func (s *Service) refuse(ctx context.Context, w protobuf.Writer, stream p2p.Stream, cause error) error { + status := pb.Status_REJECTED + if errors.Is(cause, errFull) { + status = pb.Status_FULL + } + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: status}); err != nil { + _ = stream.Reset() + return fmt.Errorf("write ack: %w", err) + } + _ = stream.FullClose() + s.logger.Debug("join refused", "status", status, "error", cause) + return nil +} + +// handleFrame validates a Broadcast arriving at the broker per SWIP-74 and +// SWIP-87 and accepts, drops or punishes it. A returned error is a violation. +func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { + principal := co.spec.Principal + admin := bytes.Equal(m.owner, principal) + role, named, cued, cursor := co.view(m, f.Kind) + + // 0. a stream being closed — revoked, or ended by an EOS — is not read: what + // it still sends was sent before it read the roster or the EOS + if role == roleRevoked || role == roleEnded { + s.dropClosing(co, role) + return nil + } + // an AUTH from a spectator a roster had named on this stream, which a later + // roster no longer names, raced its revocation: the stream is closed, the + // AUTH dropped, nothing punished + if role == roleSpectator && f.Kind == pb.Kind_AUTH && !named && cued { + co.mtx.Lock() + defer co.mtx.Unlock() + s.revokeSpectator(co, m) + return nil + } + // 1. a publisher stream sends publications and heartbeats — the admin's also + // its service messages — a pending stream its AUTH, a spectator stream its + // AUTH once a roster names it; anything else is not what the stream may send + if !mayEmit(role, admin, named, f.Kind) { + s.count(&co.counters.wrongStream, "wrong_stream") + return fmt.Errorf("%w: %w", errViolation, errWrongStream) + } + // 2. a kind this SWIP does not define is dropped + switch f.Kind { + case pb.Kind_DATA, pb.Kind_AUTH, pb.Kind_EOS, pb.Kind_ROSTER: + default: + s.count(&co.counters.unknownKind, "unknown_kind") + return nil + } + // 3. the challenge must be the stream's + if !bytes.Equal(f.Challenge, m.challenge) { + s.count(&co.counters.wrongChallenge, "wrong_challenge") + return nil + } + // 4. a DATA or ROSTER frame below its feed's cursor is a retransmit + if (f.Kind == pb.Kind_DATA || f.Kind == pb.Kind_ROSTER) && f.Index < cursor { + s.count(&co.counters.retransmit, "retransmit") + return nil + } + // 5. the chunk must be its owner's at the id derived from the frame: the + // stream's owner for DATA and AUTH, the principal for the service kinds; + // an AUTH or EOS is moreover the empty chunk at index 0. The signature check + // runs outside the lock, which is why the frame is applied in a second step. + owner := m.owner + if isServiceKind(f.Kind) { + owner = principal + } + c, err := verify(f, co.spec.Topic, owner) + if err != nil { + s.count(&co.counters.invalidSOC, "invalid_soc") + return fmt.Errorf("%w: %w", errViolation, err) + } + // 6. a ROSTER must decode; one that does not changes nothing + var roster [][]byte + if f.Kind == pb.Kind_ROSTER { + var ok bool + if roster, ok = decodeRosterMsg(payload(c)); !ok { + s.count(&co.counters.invalidRoster, "invalid_roster") + return nil + } + } + s.apply(co, m, f, roster) + return nil +} + +// apply applies a validated frame to the cohort. Every stream it ends, it ends +// by a request to the stream's handler, which never blocks, so it all runs +// under one lock. +func (s *Service) apply(co *cohort, m *member, f *pb.Broadcast, roster [][]byte) { + co.mtx.Lock() + defer co.mtx.Unlock() + + // a roster revoking the owner, or an EOS, was accepted since the checks + if r := m.role; r == roleRevoked || r == roleEnded { + s.dropClosing(co, r) + return + } + switch f.Kind { + case pb.Kind_AUTH: + switch { + case m.role == roleSpectator && !co.rostered(m.owner): + // revoked since the checks + s.revokeSpectator(co, m) + return + case m.role != rolePublisher: + // the auth of a pending stream or of a spectator a roster names + s.countQueueResets(co, co.upgrade(m, s.opts.MaxPublisherStreams)) + } + // on a publisher stream, a heartbeat + co.inactiveSince = time.Now() + + case pb.Kind_DATA: + k := string(m.owner) + if f.Index < co.cursors[k] { + // raced with another publisher stream of the same owner + s.count(&co.counters.retransmit, "retransmit") + return + } + co.cursors[k] = f.Index + 1 + co.inactiveSince = time.Now() + s.metrics.Delivered.Inc() + s.countQueueResets(co, co.deliver(f, m)) + + case pb.Kind_ROSTER: + if f.Index < co.rosterCursor { + s.count(&co.counters.retransmit, "retransmit") + return + } + co.rosterCursor = f.Index + 1 + co.inactiveSince = time.Now() + s.countQueueResets(co, co.applyRoster(f, roster, m)) + + case pb.Kind_EOS: + // the channel is ended: every stream is told — pending ones too — and + // closed, and nothing more is read from any of them; the cohort is + // forgotten when the last of them has left, and a Join meanwhile + // creates a fresh one + co.ended = true + overflow := co.deliver(f, m) + for _, o := range co.members() { + if o.role == rolePending && !o.enqueue(f) && o.requestReset() { + overflow++ + } + o.role = roleEnded + o.requestClose() + } + s.countQueueResets(co, overflow) + } +} + +// mayEmit reports whether a stream of the role may send a frame of the kind +// (step 1); named is whether the latest roster names the stream's owner. +// Kinds the broker does not define are left to step 2. +func mayEmit(r role, admin, named bool, kind pb.Kind) bool { + switch r { + case rolePublisher: + return !isServiceKind(kind) || admin + case rolePending: + return kind == pb.Kind_AUTH + case roleSpectator: + return named && kind == pb.Kind_AUTH + } + return false +} + +// dropClosing counts a frame from a stream being closed, which is not read. +func (s *Service) dropClosing(co *cohort, r role) { + if r == roleEnded { + s.count(&co.counters.ended, "ended") + return + } + s.count(&co.counters.revoked, "revoked") +} + +// countQueueResets counts the streams reset because their queue was full. +func (s *Service) countQueueResets(co *cohort, n int) { + for range n { + s.count(&co.counters.queueReset, "queue_reset") + } +} + +// revokeSpectator closes a spectator whose AUTH raced the roster revoking it. +// Must be called with co.mtx held. +func (s *Service) revokeSpectator(co *cohort, m *member) { + if m.role == roleSpectator { + m.role = roleRevoked + co.spectators-- + } + s.count(&co.counters.revoked, "revoked") + m.requestClose() +} + +// join attaches a stream to the cohort named by the spec, creating the cohort +// if no live one has it, and returns leave, which detaches the stream again: +// the stream's handler calls it once, when it returns. An error wrapping +// errFull means a capacity bound was hit. +func (s *Service) join(peer swarm.Address, join *pb.Join) (co *cohort, m *member, leave func(), err error) { + key, err := join.Cohort.Marshal() + if err != nil { + return nil, nil, nil, fmt.Errorf("marshal cohort spec: %w", err) + } + k, pk := string(key), peer.ByteString() + + s.mtx.Lock() + defer s.mtx.Unlock() + select { + case <-s.quit: + return nil, nil, nil, errSessionClosed + default: + } + + pc := s.peers[pk] + if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerConnection { + return nil, nil, nil, fmt.Errorf("too many cohorts per peer: %w", errFull) + } + if pc[k] >= s.opts.MaxPeerStreamsPerCohort { + return nil, nil, nil, fmt.Errorf("too many streams per peer per cohort: %w", errFull) + } + + co, ok := s.cohorts[k] + if ok { + // a cohort its admin's EOS ended is being forgotten: the Join creates + // a fresh one + co.mtx.Lock() + if co.ended { + s.removeCohort(co) + ok = false + } + co.mtx.Unlock() + } + if !ok { + if len(s.cohorts) >= s.opts.MaxBrokerCohorts { + return nil, nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) + } + co = newCohort(k, join.Cohort) + co.inactivity = time.AfterFunc(s.opts.InactiveTimeout, func() { s.reclaim(co) }) + s.cohorts[k] = co + s.metrics.Cohorts.Set(float64(len(s.cohorts))) + } + + co.mtx.Lock() + defer co.mtx.Unlock() + + // a stream owned by the admin is pending, outside the fan-out bound; any other + // is a spectator, within it — except that a rostered owner finding the set + // full is pending too: it receives the roster that names it and nothing + // else until its AUTH, so the bound is not bypassed by declaring a public + // address. A stream with no owner is a spectator that no roster can name. + role := roleSpectator + full := co.spectators >= s.opts.MaxSubsPerCohort + if bytes.Equal(join.Owner, join.Cohort.Principal) || full && co.rostered(join.Owner) { + role = rolePending + } + if role == roleSpectator && full { + if len(co.attached) == 0 { + s.removeCohort(co) + } + return nil, nil, nil, fmt.Errorf("too many subscribers per cohort: %w", errFull) + } + + challenge := make([]byte, ChallengeSize) + if _, err := rand.Read(challenge); err != nil { + if len(co.attached) == 0 { + s.removeCohort(co) + } + return nil, nil, nil, fmt.Errorf("challenge: %w", err) + } + m = newMember(role, join.Owner, challenge, s.opts.QueueCap) + if role == rolePending { + // a rostered owner receives the roster that names it at once: its cue + if !bytes.Equal(join.Owner, join.Cohort.Principal) { + m.queue <- co.rosterFrame + m.given = co.rosterFrame + } + } else { + co.spectators++ + // the latest roster is the first delivery on a receiving stream; one + // naming the owner is the cue for its AUTH + if co.rosterFrame != nil { + m.queue <- co.rosterFrame + m.cued = co.rostered(join.Owner) + } + } + co.attached[m] = struct{}{} + + if pc == nil { + pc = make(map[string]int) + s.peers[pk] = pc + } + pc[k]++ + + // leave detaches the stream: the peer's stream count, the member, and the + // cohort, once no stream is attached to it + leave = func() { + s.mtx.Lock() + defer s.mtx.Unlock() + if pc := s.peers[pk]; pc != nil { + if pc[k]--; pc[k] <= 0 { + delete(pc, k) + } + if len(pc) == 0 { + delete(s.peers, pk) + } + } + + co.mtx.Lock() + defer co.mtx.Unlock() + delete(co.attached, m) + if m.role == roleSpectator { + co.spectators-- + } + if len(co.attached) == 0 && s.cohorts[k] == co { + s.removeCohort(co) + } + } + return co, m, leave, nil +} + +// removeCohort forgets the cohort. Must be called with s.mtx and co.mtx held. +func (s *Service) removeCohort(co *cohort) { + delete(s.cohorts, co.key) + co.inactivity.Stop() + s.metrics.Cohorts.Set(float64(len(s.cohorts))) +} + +// reclaim forgets a cohort on which no frame was accepted for the inactivity +// deadline, and asks every stream in it to reset. +func (s *Service) reclaim(co *cohort) { + s.mtx.Lock() + defer s.mtx.Unlock() + if s.cohorts[co.key] != co { + return + } + + co.mtx.Lock() + defer co.mtx.Unlock() + if idle := time.Since(co.inactiveSince); idle < s.opts.InactiveTimeout { + co.inactivity.Reset(s.opts.InactiveTimeout - idle) + return + } + s.removeCohort(co) + for _, m := range co.members() { + m.requestReset() + } + s.logger.Debug("cohort reclaimed", "error", errCohortReclaimed) +} + +// authExpired ends a stream still pending at its auth deadline, and reports +// whether it was: a stream that authenticated, or began closing, before the +// deadline is left alone. +func (s *Service) authExpired(peer swarm.Address, co *cohort, m *member) bool { + if !co.expire(m) { + return false + } + s.count(&co.counters.authTimeout, "auth_timeout") + // a rostered owner admitted past a full fan-out set that does not + // authenticate has cost what a FULL would have: no blocklist + if s.blocklister != nil && bytes.Equal(m.owner, co.spec.Principal) { + if err := s.blocklister.Blocklist(peer, s.opts.InvalidAuthBlocklistDuration, "bps auth timeout"); err != nil { + s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) + } + } + return true +} + +// Counters returns the counters of the live cohort named by the spec. +func (s *Service) Counters(spec CohortSpec) (Counters, bool) { + key, err := spec.proto().Marshal() + if err != nil { + return Counters{}, false + } + s.mtx.Lock() + co, ok := s.cohorts[string(key)] + s.mtx.Unlock() + if !ok { + return Counters{}, false + } + return co.counters.snapshot(), true +} + +type role int + +const ( + roleSpectator role = iota + 1 // receives; sends nothing but, once a roster names it, its AUTH + rolePending // declared the principal — or a rostered address at a full fan-out set — no AUTH yet + rolePublisher // authenticated by its AUTH: publishes, and receives what others publish + roleRevoked // a publisher stream a roster dropped, or timed out pending: being closed, not read + roleEnded // a stream of a cohort its admin's EOS ended: being closed, not read +) + +// receives reports whether deliveries are enqueued on a stream of the role. +func (r role) receives() bool { return r == roleSpectator || r == rolePublisher } + +type cohort struct { + key string + spec *pb.CohortSpec + inactivity *time.Timer + counters counters + + mtx sync.Mutex + cursors map[string]uint64 // by publisher: expect the next DATA feed update at this index or above; 0 = none yet + roster [][]byte // the latest accepted roster, the principal excluded + rosterCursor uint64 // expect the next ROSTER feed update at this index or above; 0 = none yet + rosterFrame *pb.Broadcast // the latest accepted ROSTER frame: the first delivery + inactiveSince time.Time + attached map[*member]struct{} // the streams attached to the cohort + spectators int // the fan-out set + authSeq uint64 // orders publisher streams by their auth + ended bool // the admin's EOS was accepted +} + +// newCohort returns an empty cohort for the spec, keyed by its canonical +// serialisation. The caller arms its inactivity timer. +func newCohort(key string, spec *pb.CohortSpec) *cohort { + return &cohort{ + key: key, + spec: spec, + attached: make(map[*member]struct{}), + cursors: make(map[string]uint64), + inactiveSince: time.Now(), + } +} + +// members returns the streams attached to the cohort. Must be called with +// co.mtx held. +func (co *cohort) members() []*member { + ms := make([]*member, 0, len(co.attached)) + for m := range co.attached { + ms = append(ms, m) + } + return ms +} + +// rostered reports whether the latest roster names the address. Must be +// called with co.mtx held. +func (co *cohort) rostered(a []byte) bool { + return contains(co.roster, a) +} + +// view returns what the checks before the signature need of a stream: its +// role, whether the roster names its owner, whether a roster naming it was +// enqueued on it, and the cursor of the feed a frame of the kind is on. +func (co *cohort) view(m *member, kind pb.Kind) (r role, named, cued bool, cursor uint64) { + co.mtx.Lock() + defer co.mtx.Unlock() + switch kind { + case pb.Kind_DATA: + cursor = co.cursors[string(m.owner)] + case pb.Kind_ROSTER: + cursor = co.rosterCursor + } + return m.role, co.rostered(m.owner), m.cued, cursor +} + +// expire marks a stream still pending at its auth deadline as ending, so that +// an AUTH arriving now is not accepted, and reports whether it was pending. +func (co *cohort) expire(m *member) bool { + co.mtx.Lock() + defer co.mtx.Unlock() + if m.role != rolePending { + return false + } + m.role = roleRevoked + return true +} + +// deliver enqueues an accepted frame on every receiving stream — for DATA, +// every one not owned by the sender's owner — and returns how many streams +// were asked to reset because their queue was full. A ROSTER or EOS reaches +// the sender too: where it sits in the sender's stream is where it took +// effect, and every receiver, the admin included, admits by the rosters it +// receives. The send is non-blocking, so that one slow receiver cannot stall +// the cohort. Must be called with co.mtx held. +func (co *cohort) deliver(f *pb.Broadcast, sender *member) (overflow int) { + for m := range co.attached { + if !m.role.receives() || m.resetting || f.Kind == pb.Kind_DATA && bytes.Equal(m.owner, sender.owner) { + continue + } + if !m.enqueue(f) && m.requestReset() { + overflow++ + } + } + return overflow +} + +// upgrade makes a pending stream, or a spectator stream a roster names, a +// publisher stream; beyond the bound on the publisher streams one address may +// hold, the oldest of them is asked to reset. It returns 1 if the stream's +// queue could not take the latest roster, and 0 otherwise. Must be called with +// co.mtx held. +func (co *cohort) upgrade(m *member, maxStreams int) (overflow int) { + if m.role == rolePending { + // a stream that starts receiving gets the latest roster first, unless + // it was given it as its cue + if co.rosterFrame != nil && m.given != co.rosterFrame && !m.enqueue(co.rosterFrame) && m.requestReset() { + overflow++ + } + } else { + // a spectator leaves the fan-out set; it has the roster already + co.spectators-- + } + m.role = rolePublisher + co.authSeq++ + m.authSeq = co.authSeq + + var same []*member + for o := range co.attached { + if o.role == rolePublisher && !o.resetting && bytes.Equal(o.owner, m.owner) { + same = append(same, o) + } + } + for len(same) > maxStreams { + oldest := 0 + for i, o := range same { + if o.authSeq < same[oldest].authSeq { + oldest = i + } + } + same[oldest].requestReset() + same = append(same[:oldest], same[oldest+1:]...) + } + return overflow +} + +// applyRoster makes an accepted roster the cohort's: it is delivered — to its +// sender too — and every publisher stream of an owner it drops is revoked: +// written the roster, then closed, what it sends meanwhile dropped; the +// cursors of dropped owners are forgotten. A spectator it names may send +// its AUTH. It returns how many streams were asked to reset because their +// queue was full. Must be called with co.mtx held. +func (co *cohort) applyRoster(f *pb.Broadcast, roster [][]byte, sender *member) (overflow int) { + principal := co.spec.Principal + co.roster = roster + co.rosterFrame = f + overflow = co.deliver(f, sender) + for m := range co.attached { + if bytes.Equal(m.owner, principal) { + continue + } + switch { + case m.role == roleSpectator && co.rostered(m.owner): + // delivered above: the cue for its AUTH + m.cued = true + case m.role == rolePending && co.rostered(m.owner): + // a rostered pending stream is given the latest roster + m.given = f + if !m.enqueue(f) && m.requestReset() { + overflow++ + } + case m.role == rolePending: + // revoked before its auth: told, then closed + m.role = roleRevoked + if !m.enqueue(f) { + if m.requestReset() { + overflow++ + } + continue + } + m.requestClose() + case m.role == rolePublisher && !co.rostered(m.owner): + // revoked: it was delivered this roster above; closed once that + // is written + m.role = roleRevoked + m.requestClose() + } + } + for k := range co.cursors { + if a := []byte(k); !bytes.Equal(a, principal) && !co.rostered(a) { + delete(co.cursors, k) + } + } + return overflow +} + +// member is one stream attached to a cohort. Its handler owns the stream and +// is the only one that closes or resets it; anyone else asks, by +// requestClose or requestReset. The fields after the requests are guarded by +// cohort.mtx. +type member struct { + owner []byte // nil for a stream that can only listen + challenge []byte + authDeadline bool // admitted pending: the auth deadline applies + queue chan *pb.Broadcast + closeReq chan struct{} // one slot: write the queue, then close + resetReq chan struct{} // one slot: reset at once; takes precedence + + role role + authSeq uint64 + cued bool // a roster naming the owner was enqueued on this spectator stream + given *pb.Broadcast // the roster last enqueued on a rostered pending stream + resetting bool // a reset was asked for: nothing more is enqueued +} + +// newMember returns the member of a stream joining in the role, with its +// outbound queue and its two requests. +func newMember(r role, owner, challenge []byte, queueCap int) *member { + return &member{ + owner: owner, + challenge: challenge, + authDeadline: r == rolePending, + queue: make(chan *pb.Broadcast, queueCap), + closeReq: make(chan struct{}, 1), + resetReq: make(chan struct{}, 1), + role: r, + } +} + +// requestClose asks the stream's handler to write what is queued, then close +// the stream, within the flush deadline. It never blocks, and asking again +// changes nothing. +func (m *member) requestClose() { request(m.closeReq) } + +// requestReset asks the stream's handler to reset the stream at once, and +// keeps anything more from being enqueued on it. It never blocks; it reports +// whether this was the first ask. Must be called with cohort.mtx held. +func (m *member) requestReset() bool { + if m.resetting { + return false + } + m.resetting = true + request(m.resetReq) + return true +} + +// request fills a one-slot request channel, unless it is full already. The +// channel is never closed, so no ask can panic, however often it is made. +func request(c chan struct{}) { + select { + case c <- struct{}{}: + default: + } +} + +// enqueue adds a frame to the stream's outbound queue, and reports false if +// the queue is full. +func (m *member) enqueue(f *pb.Broadcast) bool { + select { + case m.queue <- f: + return true + default: + return false + } +} diff --git a/pkg/bps/bps_fuzz_test.go b/pkg/bps/bps_fuzz_test.go new file mode 100644 index 00000000000..5dc86eb2496 --- /dev/null +++ b/pkg/bps/bps_fuzz_test.go @@ -0,0 +1,163 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "testing" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// FuzzVerify fuzzes the peer-supplied Broadcast validation paths used by BPS — +// verify at a known owner, and recoverOwner at a subscriber — over every kind. +// It exercises malformed frames and valid signed chunks, asserts neither +// panics on hostile input, and that a chunk valid at an owner recovers that +// owner. +func FuzzVerify(f *testing.F) { + priv, err := crypto.GenerateSecp256k1Key() + if err != nil { + f.Fatal(err) + } + signer := crypto.NewDefaultSigner(priv) + principal, err := signer.EthereumAddress() + if err != nil { + f.Fatal(err) + } + + topic := make([]byte, swarm.HashSize) + copy(topic, "bps-fuzz-topic") + challenge := make([]byte, ChallengeSize) + copy(challenge, "0123456789abcdef0123456789abcdef") + + dataID, err := ID(KindData, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + payload, err := cac.New([]byte("fuzz payload")) + if err != nil { + f.Fatal(err) + } + validChunk, err := soc.New(dataID, payload).Sign(signer) + if err != nil { + f.Fatal(err) + } + + authID, err := ID(KindAuth, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + authPayload, err := cac.New(nil) + if err != nil { + f.Fatal(err) + } + authChunk, err := soc.New(authID, authPayload).Sign(signer) + if err != nil { + f.Fatal(err) + } + if len(authChunk.Data()) != swarm.SocMinChunkSize { + f.Fatal("auth seed not the empty auth layout") + } + + rosterID, err := ID(KindRoster, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + rosterPayload, err := EncodeRosterMsg([][]byte{principal.Bytes()}) + if err != nil { + f.Fatal(err) + } + rosterCAC, err := cac.New(rosterPayload) + if err != nil { + f.Fatal(err) + } + rosterChunk, err := soc.New(rosterID, rosterCAC).Sign(signer) + if err != nil { + f.Fatal(err) + } + + eosID, err := ID(KindEOS, topic, challenge, 0) + if err != nil { + f.Fatal(err) + } + eosChunk, err := soc.New(eosID, authPayload).Sign(signer) + if err != nil { + f.Fatal(err) + } + + f.Add(validChunk.Data(), uint8(pb.Kind_DATA), challenge, uint64(0), topic, principal.Bytes()) + f.Add(rosterChunk.Data(), uint8(pb.Kind_ROSTER), challenge, uint64(0), topic, principal.Bytes()) + f.Add(eosChunk.Data(), uint8(pb.Kind_EOS), challenge, uint64(0), topic, principal.Bytes()) + f.Add(authChunk.Data(), uint8(pb.Kind_AUTH), challenge, uint64(0), topic, principal.Bytes()) + f.Add([]byte{}, uint8(pb.Kind_DATA), make([]byte, ChallengeSize), uint64(0), topic, principal.Bytes()) + f.Add(make([]byte, swarm.HashSize), uint8(pb.Kind_DATA), []byte{}, uint64(0), topic[:31], principal.Bytes()[:19]) + + f.Fuzz(func(t *testing.T, socData []byte, kind uint8, challenge []byte, index uint64, topic []byte, principal []byte) { + if kind > uint8(pb.Kind_ROSTER) { + kind = uint8(pb.Kind_KIND_UNSPECIFIED) + } + frame := &pb.Broadcast{Soc: socData, Kind: pb.Kind(kind), Challenge: challenge, Index: index} + parsed, parseErr := parseFrame(frame, topic) + _, _ = decodeRosterMsg(socData) + if _, err := verify(frame, topic, principal); err != nil { + return + } + // a chunk valid at an owner parses, and recovers that owner + if parseErr != nil || !bytes.Equal(parsed.OwnerAddress(), principal) { + t.Fatalf("valid at %x, parsed %v", principal, parseErr) + } + // and is a single-owner chunk at keccak256(id | owner) + addr, err := soc.CreateAddress(parsed.ID(), principal) + if err != nil { + t.Fatal(err) + } + if !soc.Valid(swarm.NewChunk(addr, socData)) { + t.Fatal("valid at an owner, not a single-owner chunk at its address") + } + }) +} + +// FuzzRoster fuzzes the decoding of a ROSTER chunk's payload, which a peer +// supplies: it never panics, every address it returns is 20 bytes, and what +// it returns encodes and decodes to itself. +func FuzzRoster(f *testing.F) { + one, err := EncodeRosterMsg([][]byte{make([]byte, crypto.AddressSize)}) + if err != nil { + f.Fatal(err) + } + f.Add(one) + f.Add([]byte{}) + f.Add([]byte{0x0a, 0x02, 0x01, 0x02}) + f.Add([]byte{0x0a, 0xff, 0xff, 0xff, 0xff, 0x0f}) + + f.Fuzz(func(t *testing.T, b []byte) { + roster, ok := decodeRosterMsg(b) + if !ok { + return + } + for _, a := range roster { + if len(a) != crypto.AddressSize { + t.Fatalf("address of %d bytes", len(a)) + } + } + encoded, err := EncodeRosterMsg(roster) + if err != nil { + t.Fatal(err) + } + again, ok := decodeRosterMsg(encoded) + if !ok || len(again) != len(roster) { + t.Fatalf("round trip: %d of %d (%t)", len(again), len(roster), ok) + } + for i := range roster { + if !bytes.Equal(again[i], roster[i]) { + t.Fatalf("round trip: entry %d differs", i) + } + } + }) +} diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go new file mode 100644 index 00000000000..8eee30cb2bb --- /dev/null +++ b/pkg/bps/bps_test.go @@ -0,0 +1,807 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps_test + +import ( + "bytes" + "context" + "errors" + "math" + "sync" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/p2p/streamtest" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +const waitFor = 5 * time.Second + +type blocklister struct { + mu sync.Mutex + peers map[string]string + durations map[string]time.Duration +} + +func (b *blocklister) NetworkStatus() p2p.NetworkStatus { return p2p.NetworkStatusAvailable } + +func (b *blocklister) Blocklist(overlay swarm.Address, d time.Duration, reason string) error { + b.mu.Lock() + defer b.mu.Unlock() + b.peers[overlay.ByteString()] = reason + if b.durations == nil { + b.durations = make(map[string]time.Duration) + } + b.durations[overlay.ByteString()] = d + return nil +} + +func (b *blocklister) blocklisted(overlay swarm.Address) bool { + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.peers[overlay.ByteString()] + return ok +} + +type env struct { + t *testing.T + broker *bps.Service + brokerAddr swarm.Address + bl *blocklister + signer crypto.Signer + principal []byte + topic []byte +} + +func newEnv(t *testing.T, o bps.Options) *env { + t.Helper() + bl := &blocklister{peers: make(map[string]string)} + broker := bps.New(nil, bl, true, log.Noop, o) + t.Cleanup(func() { _ = broker.Close() }) + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + admin, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + topic := make([]byte, swarm.HashSize) + copy(topic, "bps-test-topic") + return &env{t: t, broker: broker, brokerAddr: swarm.RandAddress(t), bl: bl, signer: signer, principal: admin.Bytes(), topic: topic} +} + +// client returns a node whose streams reach the broker from overlay. +func (e *env) client(overlay swarm.Address) (*bps.Service, *streamtest.Recorder) { + rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(overlay)) + return bps.New(rec, nil, false, log.Noop, bps.Options{}), rec +} + +func (e *env) join(c *bps.Service, addr []byte) bps.Session { + e.t.Helper() + s, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Owner: addr}) + if err != nil { + e.t.Fatal(err) + } + e.t.Cleanup(func() { _ = s.Close() }) + return s +} + +func (e *env) subscribe() bps.Session { + e.t.Helper() + c, _ := e.client(swarm.RandAddress(e.t)) + addr := make([]byte, 20) + copy(addr, swarm.RandAddress(e.t).Bytes()) + return e.join(c, addr) +} + +// pending joins as the principal and sends nothing: a pending stream. +func (e *env) pending() (bps.Session, swarm.Address) { + e.t.Helper() + overlay := swarm.RandAddress(e.t) + c, _ := e.client(overlay) + return e.join(c, e.principal), overlay +} + +// publisher joins as the principal and authenticates the stream with its AUTH. +func (e *env) publisher() (bps.Session, swarm.Address) { + e.t.Helper() + s, overlay := e.pending() + e.publish(s, bps.KindAuth, 0, nil) + return s, overlay +} + +func (e *env) chunk(signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { + e.t.Helper() + id, err := bps.ID(kind, e.topic, challenge, index) + if err != nil { + e.t.Fatal(err) + } + ch, err := cac.New(payload) + if err != nil { + e.t.Fatal(err) + } + s, err := soc.New(id, ch).Sign(signer) + if err != nil { + e.t.Fatal(err) + } + return s.Data() +} + +func (e *env) publish(s bps.Session, kind bps.Kind, index uint64, payload []byte) []byte { + e.t.Helper() + c := e.chunk(e.signer, kind, s.Challenge(), index, payload) + if err := s.Publish(context.Background(), kind, index, c); err != nil { + e.t.Fatal(err) + } + return c +} + +func (e *env) counters() bps.Counters { + e.t.Helper() + c, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}) + if !ok { + e.t.Fatal("no live cohort") + } + return c +} + +func expectMessage(t *testing.T, s bps.Session, index uint64, chunk []byte) { + t.Helper() + select { + case m := <-s.Messages(): + if m.Index != index || !bytes.Equal(m.SOC, chunk) { + t.Fatalf("got index %d %x, want %d %x", m.Index, m.SOC, index, chunk) + } + case <-time.After(waitFor): + t.Fatalf("timed out waiting for index %d", index) + } +} + +func expectNoMessage(t *testing.T, s bps.Session) { + t.Helper() + select { + case m := <-s.Messages(): + t.Fatalf("unexpected message at index %d", m.Index) + case <-time.After(100 * time.Millisecond): + } +} + +func expectDone(t *testing.T, s bps.Session) { + t.Helper() + select { + case <-s.Done(): + case <-time.After(waitFor): + t.Fatal("stream not ended") + } +} + +func eventually(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(waitFor) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not met") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestBroadcast(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, _ := e.publisher() + + if len(pub.Challenge()) != bps.ChallengeSize || bytes.Equal(pub.Challenge(), sub.Challenge()) { + t.Fatal("want a distinct 32-byte challenge per stream") + } + + // the AUTH that authenticated the stream was not delivered; sent again it is a heartbeat + e.publish(pub, bps.KindAuth, 0, nil) + c0 := e.publish(pub, bps.KindData, 0, []byte("hello cohort")) + expectMessage(t, sub, 0, c0) + expectNoMessage(t, pub) +} + +func TestCursor(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, _ := e.publisher() + + c5 := e.publish(pub, bps.KindData, 5, []byte("five")) // gaps are allowed + expectMessage(t, sub, 5, c5) + e.publish(pub, bps.KindData, 3, []byte("three")) // retransmit + e.publish(pub, bps.KindData, 5, []byte("five")) // retransmit + c6 := e.publish(pub, bps.KindData, 6, []byte("six")) + expectMessage(t, sub, 6, c6) + if got := e.counters().Retransmit; got != 2 { + t.Fatalf("retransmit %d, want 2", got) + } +} + +func TestPublisherReconnects(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + + pub1, _ := e.publisher() + for i := uint64(0); i < 3; i++ { + expectMessage(t, sub, i, e.publish(pub1, bps.KindData, i, []byte{byte(i)})) + } + _ = pub1.Close() + expectDone(t, pub1) + + // the cohort and its cursor persist; the new stream authenticates with its AUTH and + // an update below the cursor is counted as a retransmit + pub2, _ := e.publisher() + if bytes.Equal(pub1.Challenge(), pub2.Challenge()) { + t.Fatal("challenge reused") + } + e.publish(pub2, bps.KindData, 1, []byte{1}) + c3 := e.publish(pub2, bps.KindData, 3, []byte{3}) + expectMessage(t, sub, 3, c3) + if got := e.counters().Retransmit; got != 1 { + t.Fatalf("retransmit %d, want 1", got) + } +} + +func TestTwoPublisherStreams(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub1, _ := e.publisher() + pub2, _ := e.publisher() + + c0 := e.publish(pub1, bps.KindData, 0, []byte("a")) + expectMessage(t, sub, 0, c0) + e.publish(pub2, bps.KindData, 0, []byte("a")) // signed for its own stream, delivered once + c1 := e.publish(pub2, bps.KindData, 1, []byte("b")) + expectMessage(t, sub, 1, c1) + expectNoMessage(t, pub1) +} + +func TestViolations(t *testing.T) { + t.Parallel() + + other, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + otherSigner := crypto.NewDefaultSigner(other) + + t.Run("publish on a subscriber session", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + c := e.chunk(e.signer, bps.KindData, sub.Challenge(), 0, []byte("x")) + if err := sub.Publish(context.Background(), bps.KindData, 0, c); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("got %v, want %v", err, bps.ErrNotPublisher) + } + }) + + t.Run("publication from a subscriber stream", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + overlay := swarm.RandAddress(t) + keep := e.subscribe() + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Owner: swarm.RandAddress(t).Bytes()[:20]}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + c := e.chunk(e.signer, bps.KindData, ack.Challenge, 0, []byte("x")) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ack.Challenge, Soc: c}); err != nil { + t.Fatal(err) + } + if err := r.ReadMsg(&pb.Broadcast{}); err == nil { + t.Fatal("stream not ended") + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().WrongStream; got != 1 { + t.Fatalf("wrong_stream %d, want 1", got) + } + expectNoMessage(t, keep) + }) + + // a session refuses to send these (see TestPublishChecks); a peer on a raw + // stream can, and is punished for it + for _, tc := range []struct { + name string + frame func(e *env, challenge []byte) *pb.Broadcast + }{ + {"wrong signer", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ch, Soc: e.chunk(otherSigner, bps.KindData, ch, 0, []byte("x"))} + }}, + {"signed for another challenge", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ch, Soc: e.chunk(e.signer, bps.KindData, make([]byte, 32), 0, []byte("x"))} + }}, + {"index not the signed one", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ch, Index: 1, Soc: e.chunk(e.signer, bps.KindData, ch, 0, []byte("x"))} + }}, + {"auth relabelled as data", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ch, Soc: e.chunk(e.signer, bps.KindAuth, ch, 0, nil)} + }}, + {"auth with payload", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ch, Soc: e.chunk(e.signer, bps.KindAuth, ch, 0, []byte("x"))} + }}, + {"auth at index 1", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ch, Index: 1, Soc: e.chunk(e.signer, bps.KindAuth, ch, 1, nil)} + }}, + {"index that would overflow the cursor", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ch, Index: math.MaxUint64, Soc: e.chunk(e.signer, bps.KindData, ch, math.MaxUint64, []byte("x"))} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Owner: e.principal}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + auth := e.chunk(e.signer, bps.KindAuth, ack.Challenge, 0, nil) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}); err != nil { + t.Fatal(err) + } + if err := w.WriteMsg(tc.frame(e, ack.Challenge)); err != nil { + t.Fatal(err) + } + if err := r.ReadMsg(&pb.Broadcast{}); err == nil { + t.Fatal("stream not ended") + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().InvalidSOC; got != 1 { + t.Fatalf("invalid_soc %d, want 1", got) + } + expectNoMessage(t, sub) + }) + } + + // a pending stream may send nothing but its AUTH: a valid update, or a kind the + // broker does not know, is a violation before any signature is checked + for _, tc := range []struct { + name string + frame func(e *env, challenge []byte) *pb.Broadcast + }{ + {"data before auth", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ch, Index: 0, Soc: e.chunk(e.signer, bps.KindData, ch, 0, []byte("x"))} + }}, + {"unknown kind before auth", func(e *env, ch []byte) *pb.Broadcast { + return &pb.Broadcast{Kind: pb.Kind(7), Challenge: ch, Soc: []byte("roster")} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Owner: e.principal}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + if err := w.WriteMsg(tc.frame(e, ack.Challenge)); err != nil { + t.Fatal(err) + } + if err := r.ReadMsg(&pb.Broadcast{}); err == nil { + t.Fatal("stream not ended") + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + got := e.counters() + if got.WrongStream != 1 || got.InvalidSOC != 0 || got.UnknownKind != 0 { + t.Fatalf("counters %+v", got) + } + expectNoMessage(t, sub) + }) + } +} + +// raw opens a stream to the broker and joins with join, returning the Ack. +func (e *env) raw(join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { + e.t.Helper() + return e.rawFrom(swarm.RandAddress(e.t), join) +} + +// rawFrom is raw from the given overlay. +func (e *env) rawFrom(overlay swarm.Address, join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { + e.t.Helper() + rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(overlay)) + stream, err := rec.NewStream(context.Background(), e.brokerAddr, nil, bps.ProtocolName, bps.ProtocolVersion, bps.StreamName) + if err != nil { + e.t.Fatal(err) + } + e.t.Cleanup(func() { _ = stream.Reset() }) + w, r := protobuf.NewWriterAndReader(stream) + if err := w.WriteMsg(join); err != nil { + e.t.Fatal(err) + } + var ack pb.Ack + if err := r.ReadMsg(&ack); err != nil { + e.t.Fatal(err) + } + return w, r, &ack +} + +func (e *env) spec() *pb.CohortSpec { + return &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: e.principal} +} + +func TestDroppedFrames(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + w, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Owner: e.principal}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + write := func(f *pb.Broadcast) { + t.Helper() + if err := w.WriteMsg(f); err != nil { + t.Fatal(err) + } + } + + write(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Index: 0, Soc: e.chunk(e.signer, bps.KindAuth, ack.Challenge, 0, nil)}) + + // on a publisher stream another kind and another challenge are dropped, not punished + write(&pb.Broadcast{Kind: pb.Kind(7), Challenge: ack.Challenge, Soc: []byte("roster")}) + other := make([]byte, 32) + write(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: other, Soc: e.chunk(e.signer, bps.KindData, other, 0, []byte("old"))}) + c := e.chunk(e.signer, bps.KindData, ack.Challenge, 0, []byte("ok")) + write(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ack.Challenge, Index: 0, Soc: c}) + expectMessage(t, sub, 0, c) + + got := e.counters() + if got.UnknownKind != 1 || got.WrongChallenge != 1 || got.InvalidSOC != 0 { + t.Fatalf("counters %+v", got) + } +} + +// TestInvalidDelivery checks that a subscriber ends the session when the +// broker forwards a chunk that does not verify against the cohort spec. +func TestInvalidDelivery(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + other, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + challenge := make([]byte, bps.ChallengeSize) + bad := e.chunk(crypto.NewDefaultSigner(other), bps.KindData, challenge, 0, []byte("x")) + broker := p2p.ProtocolSpec{ + Name: bps.ProtocolName, + Version: bps.ProtocolVersion, + StreamSpecs: []p2p.StreamSpec{{ + Name: bps.StreamName, + Handler: func(ctx context.Context, _ p2p.Peer, stream p2p.Stream) error { + w, r := protobuf.NewWriterAndReader(stream) + if err := r.ReadMsgWithContext(ctx, &pb.Join{}); err != nil { + return err + } + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: challenge}); err != nil { + return err + } + if err := w.WriteMsgWithContext(ctx, &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: challenge, Soc: bad}); err != nil { + return err + } + // hold the stream until the subscriber ends it + _ = r.ReadMsgWithContext(ctx, &pb.Broadcast{}) + return nil + }, + }}, + } + rec := streamtest.New(streamtest.WithProtocols(broker), streamtest.WithBaseAddr(swarm.RandAddress(t))) + bl := &blocklister{peers: make(map[string]string)} + c := bps.New(rec, bl, false, log.Noop, bps.Options{}) + sub := e.join(c, swarm.RandAddress(t).Bytes()[:20]) + expectDone(t, sub) + if err := sub.Err(); !errors.Is(err, bps.ErrInvalidDelivery) { + t.Fatalf("got %v, want %v", err, bps.ErrInvalidDelivery) + } + // the broker is blocklisted for long + eventually(t, func() bool { return bl.blocklisted(e.brokerAddr) }) + bl.mu.Lock() + d := bl.durations[e.brokerAddr.ByteString()] + bl.mu.Unlock() + if d != bps.ForgingBrokerBlocklistDuration { + t.Fatalf("broker blocklisted for %v, want %v", d, bps.ForgingBrokerBlocklistDuration) + } +} + +func TestJoinRejected(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + for _, tc := range []struct { + name string + join *pb.Join + }{ + {"no cohort", &pb.Join{Owner: e.principal}}, + {"unspecified binding", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Principal: e.principal}, Owner: e.principal}}, + {"short topic", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic[:31], Binding: pb.TopicBinding_FEED_TOPIC, Principal: e.principal}, Owner: e.principal}}, + {"no admin", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC}, Owner: e.principal}}, + {"short addr", &pb.Join{Cohort: e.spec(), Owner: e.principal[:19]}}, + } { + _, _, ack := e.raw(tc.join) + if ack.Status != pb.Status_REJECTED || len(ack.Challenge) != 0 { + t.Fatalf("%s: got %s", tc.name, ack.Status) + } + } +} + +func TestBounds(t *testing.T) { + t.Parallel() + + t.Run("subscribers per cohort, principal admitted outside", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxSubsPerCohort: 1}) + sub := e.subscribe() + if _, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Owner: make([]byte, 20)}); ack.Status != pb.Status_FULL { + t.Fatalf("got %s, want FULL", ack.Status) + } + pub, _ := e.publisher() + expectMessage(t, sub, 0, e.publish(pub, bps.KindData, 0, []byte("x"))) + }) + + t.Run("cohorts per broker", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxBrokerCohorts: 1}) + e.subscribe() + spec := e.spec() + spec.Topic = make([]byte, 32) + if _, _, ack := e.raw(&pb.Join{Cohort: spec, Owner: e.principal}); ack.Status != pb.Status_FULL { + t.Fatalf("got %s, want FULL", ack.Status) + } + }) + + t.Run("streams and cohorts per peer", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxPeerStreamsPerCohort: 1, MaxCohortsPerConnection: 1}) + c, _ := e.client(swarm.RandAddress(t)) + e.join(c, e.principal) + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Owner: e.principal}); !errors.Is(err, bps.ErrJoinRefused) { + t.Fatalf("second stream on the cohort: got %v, want %v", err, bps.ErrJoinRefused) + } + other := make([]byte, 32) + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: other, Principal: e.principal}, Owner: e.principal}); !errors.Is(err, bps.ErrJoinRefused) { + t.Fatalf("second cohort: got %v, want %v", err, bps.ErrJoinRefused) + } + }) + + t.Run("auth timeout", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{AuthWaitTimeout: 50 * time.Millisecond}) + e.subscribe() + pub, overlay := e.pending() + expectDone(t, pub) + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().AuthTimeout; got != 1 { + t.Fatalf("auth_timeout %d, want 1", got) + } + }) + + t.Run("inactivity deadline", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{InactiveTimeout: 100 * time.Millisecond}) + sub := e.subscribe() + pub, _ := e.publisher() + expectDone(t, sub) + expectDone(t, pub) + if _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}); ok { + t.Fatal("cohort not reclaimed") + } + }) +} + +func TestVerify(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + challenge := make([]byte, 32) + copy(challenge, "challenge") + + c := e.chunk(e.signer, bps.KindData, challenge, 9, []byte("x")) + if err := bps.Verify(bps.KindData, challenge, 9, c, e.topic, e.principal); err != nil { + t.Fatal(err) + } + auth := e.chunk(e.signer, bps.KindAuth, challenge, 0, nil) + if err := bps.Verify(bps.KindAuth, challenge, 0, auth, e.topic, e.principal); err != nil { + t.Fatal(err) + } + other := make([]byte, 20) + for _, err := range []error{ + bps.Verify(bps.KindAuth, challenge, 1, e.chunk(e.signer, bps.KindAuth, challenge, 1, nil), e.topic, e.principal), + bps.Verify(bps.KindData, challenge, 9, c, e.topic, other), + bps.Verify(bps.KindAuth, challenge, 9, c, e.topic, e.principal), + bps.Verify(bps.KindData, challenge, 9, c, make([]byte, 32), e.principal), + bps.Verify(bps.KindData, challenge, 9, c[:10], e.topic, e.principal), + } { + if !errors.Is(err, bps.ErrInvalidSOC) { + t.Fatalf("got %v, want invalid soc", err) + } + } + // these say nothing of the chunk: the frame's own fields are wrong + for _, err := range []error{ + bps.Verify(bps.KindData, challenge[:31], 9, c, e.topic, e.principal), + bps.Verify(bps.KindData, challenge, math.MaxUint64, + e.chunk(e.signer, bps.KindData, challenge, math.MaxUint64, []byte("x")), e.topic, e.principal), + } { + if !errors.Is(err, bps.ErrInvalidMessage) { + t.Fatalf("got %v, want invalid message", err) + } + } +} + +// TestPublishChecks: a session refuses what the broker would punish, and tells +// the caller: nothing reaches the broker, and the stream lives on. +func TestPublishChecks(t *testing.T) { + t.Parallel() + + other, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + otherSigner := crypto.NewDefaultSigner(other) + rosterMsg := func(t *testing.T, addrs ...[]byte) []byte { + t.Helper() + b, err := bps.EncodeRosterMsg(addrs) + if err != nil { + t.Fatal(err) + } + return b + } + + for _, tc := range []struct { + name string + noAuth bool // the session has not sent its AUTH + frame func(t *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) + want error + }{ + {name: "wrong signer", frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(otherSigner, bps.KindData, ch, 0, []byte("x")) + }, want: bps.ErrInvalidSOC}, + {name: "signed for another challenge", frame: func(_ *testing.T, e *env, _ []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(e.signer, bps.KindData, make([]byte, 32), 0, []byte("x")) + }, want: bps.ErrInvalidSOC}, + {name: "index not the signed one", frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 1, e.chunk(e.signer, bps.KindData, ch, 0, []byte("x")) + }, want: bps.ErrInvalidSOC}, + {name: "auth relabelled as data", frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(e.signer, bps.KindAuth, ch, 0, nil) + }, want: bps.ErrInvalidSOC}, + {name: "not a chunk", frame: func(*testing.T, *env, []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, []byte("x") + }, want: bps.ErrInvalidSOC}, + {name: "auth with payload", noAuth: true, frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindAuth, 0, e.chunk(e.signer, bps.KindAuth, ch, 0, []byte("x")) + }, want: bps.ErrInvalidSOC}, + {name: "auth at index 1", noAuth: true, frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindAuth, 1, e.chunk(e.signer, bps.KindAuth, ch, 1, nil) + }, want: bps.ErrInvalidSOC}, + {name: "data before auth", noAuth: true, frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(e.signer, bps.KindData, ch, 0, []byte("x")) + }, want: bps.ErrInvalidMessage}, + {name: "invalid data before auth is still a message error", noAuth: true, frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(otherSigner, bps.KindData, ch, 0, []byte("x")) + }, want: bps.ErrInvalidMessage}, + {name: "unknown kind", frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.Kind(9), 0, e.chunk(e.signer, bps.KindData, ch, 0, []byte("x")) + }, want: bps.ErrInvalidMessage}, + {name: "index that would overflow the cursor", frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, math.MaxUint64, e.chunk(e.signer, bps.KindData, ch, math.MaxUint64, []byte("x")) + }, want: bps.ErrInvalidMessage}, + {name: "roster that does not decode", frame: func(_ *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindRoster, 0, e.chunk(e.signer, bps.KindRoster, ch, 0, []byte{0x0a, 0x01, 0x01}) + }, want: bps.ErrInvalidRoster}, + {name: "roster naming the admin", frame: func(t *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + t.Helper() + return bps.KindRoster, 0, e.chunk(e.signer, bps.KindRoster, ch, 0, rosterMsg(t, e.principal)) + }, want: bps.ErrInvalidRoster}, + {name: "roster naming the admin with another", frame: func(t *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + t.Helper() + return bps.KindRoster, 0, e.chunk(e.signer, bps.KindRoster, ch, 0, rosterMsg(t, make([]byte, 20), e.principal)) + }, want: bps.ErrInvalidRoster}, + {name: "roster naming an address twice", frame: func(t *testing.T, e *env, ch []byte) (bps.Kind, uint64, []byte) { + t.Helper() + a := make([]byte, 20) + return bps.KindRoster, 0, e.chunk(e.signer, bps.KindRoster, ch, 0, rosterMsg(t, a, a)) + }, want: bps.ErrInvalidRoster}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, _ := e.pending() + if !tc.noAuth { + e.publish(pub, bps.KindAuth, 0, nil) + } + kind, index, chunk := tc.frame(t, e, pub.Challenge()) + if err := pub.Publish(context.Background(), kind, index, chunk); !errors.Is(err, tc.want) { + t.Fatalf("got %v, want %v", err, tc.want) + } + // the stream lives on: the AUTH and an update still go through + if tc.noAuth { + e.publish(pub, bps.KindAuth, 0, nil) + } + expectMessage(t, sub, 0, e.publish(pub, bps.KindData, 0, []byte("ok"))) + if got := e.counters(); got.InvalidSOC != 0 || got.WrongStream != 0 || got.InvalidRoster != 0 { + t.Fatalf("counters %+v", got) + } + }) + } + + t.Run("a closed session", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + pub, _ := e.publisher() + c := e.chunk(e.signer, bps.KindData, pub.Challenge(), 0, []byte("x")) + if err := pub.Close(); err != nil { + t.Fatal(err) + } + expectDone(t, pub) + if err := pub.Publish(context.Background(), bps.KindData, 0, c); !errors.Is(err, bps.ErrSessionClosed) { + t.Fatalf("got %v, want %v", err, bps.ErrSessionClosed) + } + }) +} + +func TestJoinInvalid(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + c, _ := e.client(swarm.RandAddress(t)) + _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic[:31], Principal: e.principal}, Owner: e.principal}) + if !errors.Is(err, bps.ErrInvalidJoin) { + t.Fatalf("got %v, want %v", err, bps.ErrInvalidJoin) + } +} + +// TestJoinChallengeLength: a broker that issues a challenge of another length +// is not joined. +func TestJoinChallengeLength(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + broker := p2p.ProtocolSpec{ + Name: bps.ProtocolName, + Version: bps.ProtocolVersion, + StreamSpecs: []p2p.StreamSpec{{ + Name: bps.StreamName, + Handler: func(ctx context.Context, _ p2p.Peer, stream p2p.Stream) error { + w, r := protobuf.NewWriterAndReader(stream) + if err := r.ReadMsgWithContext(ctx, &pb.Join{}); err != nil { + return err + } + return w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: make([]byte, bps.ChallengeSize-1)}) + }, + }}, + } + rec := streamtest.New(streamtest.WithProtocols(broker), streamtest.WithBaseAddr(swarm.RandAddress(t))) + c := bps.New(rec, nil, false, log.Noop, bps.Options{}) + _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Owner: e.principal}) + if !errors.Is(err, bps.ErrProtocolError) { + t.Fatalf("got %v, want %v", err, bps.ErrProtocolError) + } +} + +func TestJoinNotBroker(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + notBroker := bps.New(nil, nil, false, log.Noop, bps.Options{}) + rec := streamtest.New(streamtest.WithProtocols(notBroker.Protocol()), streamtest.WithBaseAddr(swarm.RandAddress(t))) + c := bps.New(rec, nil, false, log.Noop, bps.Options{}) + _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Owner: e.principal}) + if err == nil { + t.Fatal("join to a non-broker succeeded") + } + records := rec.WaitRecords(t, e.brokerAddr, bps.ProtocolName, bps.ProtocolVersion, bps.StreamName, 1, 5) + eventually(t, func() bool { return errors.Is(records[0].Err(), bps.ErrNotBroker) }) +} diff --git a/pkg/bps/export_test.go b/pkg/bps/export_test.go new file mode 100644 index 00000000000..72cc25fb583 --- /dev/null +++ b/pkg/bps/export_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import "github.com/ethersphere/bee/v2/pkg/bps/pb" + +const ( + ProtocolName = protocolName + ProtocolVersion = protocolVersion + StreamName = streamName +) + +var ( + ErrNotBroker = errNotBroker + ErrJoinRefused = errJoinRefused + ErrInvalidJoin = errInvalidJoin + ErrSessionClosed = errSessionClosed + ErrProtocolError = errProtocolError + ErrInvalidDelivery = errInvalidDelivery + ErrChannelEnded = errChannelEnded +) + +// Verify checks a chunk against the frame fields it was derived from and its owner. +func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, owner []byte) error { + _, err := verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, owner) + return err +} + +// EncodeRosterMsg returns the payload of a ROSTER chunk naming the publishers. +func EncodeRosterMsg(publishers [][]byte) ([]byte, error) { + return (&pb.Roster{Publishers: publishers}).Marshal() +} diff --git a/pkg/bps/flush_test.go b/pkg/bps/flush_test.go new file mode 100644 index 00000000000..e1a2f46c080 --- /dev/null +++ b/pkg/bps/flush_test.go @@ -0,0 +1,443 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/coreos/go-semver/semver" + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +var errFlushStreamReset = errors.New("flush stream reset") + +// flushStream is a stream whose far end the test drives: the far end's frames +// are written into a pipe this end reads, and this end's writes land in a +// buffer — or, once stalled, block as on a transport whose far end has stopped +// reading, until the stream is reset or closed. It counts every Reset and +// Close, so that a test can tell the stream ended once. +type flushStream struct { + r *io.PipeReader + pw *io.PipeWriter + + mu sync.Mutex + out bytes.Buffer + stall bool + blocked int // writes that blocked + stalled chan struct{} // closed when the first write blocks + resets, closes int + reset, closed chan struct{} // closed at the first Reset, the first Close + unblock chan struct{} // closed when the stream first ends, either way +} + +func newFlushStream() *flushStream { + r, pw := io.Pipe() + return &flushStream{ + r: r, pw: pw, + stalled: make(chan struct{}), + reset: make(chan struct{}), + closed: make(chan struct{}), + unblock: make(chan struct{}), + } +} + +func (s *flushStream) Read(p []byte) (int, error) { return s.r.Read(p) } + +func (s *flushStream) Write(p []byte) (int, error) { + s.mu.Lock() + if !s.stall { + defer s.mu.Unlock() + return s.out.Write(p) + } + s.blocked++ + if s.blocked == 1 { + close(s.stalled) + } + s.mu.Unlock() + <-s.unblock + return 0, errFlushStreamReset +} + +func (s *flushStream) Reset() error { + s.mu.Lock() + defer s.mu.Unlock() + s.resets++ + if s.resets == 1 { + close(s.reset) + } + s.end() + return nil +} + +func (s *flushStream) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closes++ + if s.closes == 1 { + close(s.closed) + } + s.end() + return nil +} + +// end unblocks writes and reads when the stream first ends. Must be called +// with s.mu held. +func (s *flushStream) end() { + if s.resets+s.closes == 1 { + close(s.unblock) + _ = s.r.Close() + } +} + +// ended reports how many times the stream was reset and closed. +func (s *flushStream) ended() (resets, closes int) { + s.mu.Lock() + defer s.mu.Unlock() + return s.resets, s.closes +} + +func (s *flushStream) FullClose() error { return s.Close() } +func (s *flushStream) Headers() p2p.Headers { return nil } +func (s *flushStream) ResponseHeaders() p2p.Headers { return nil } +func (s *flushStream) Version() (*semver.Version, error) { return nil, nil } + +// stallWrites makes every later broker write block. +func (s *flushStream) stallWrites() { + s.mu.Lock() + s.stall = true + s.mu.Unlock() +} + +// frames returns what the broker has written so far, after the Ack. +func (s *flushStream) frames(t *testing.T) []*pb.Broadcast { + t.Helper() + s.mu.Lock() + r := protobuf.NewReader(bytes.NewReader(append([]byte{}, s.out.Bytes()...))) + s.mu.Unlock() + var ack pb.Ack + if err := r.ReadMsg(&ack); err != nil { + t.Fatal(err) + } + var fs []*pb.Broadcast + for { + var f pb.Broadcast + if err := r.ReadMsg(&f); err != nil { + return fs + } + fs = append(fs, &f) + } +} + +// flushEnv runs the broker's handler over flushStreams. +type flushEnv struct { + t *testing.T + svc *Service + admin crypto.Signer + spec *pb.CohortSpec +} + +func newFlushEnv(t *testing.T, o Options) *flushEnv { + t.Helper() + svc := New(nil, nil, true, log.Noop, o) + t.Cleanup(func() { _ = svc.Close() }) + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + principal, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + topic := make([]byte, swarm.HashSize) + copy(topic, "bps-flush-topic") + return &flushEnv{t: t, svc: svc, admin: signer, + spec: &pb.CohortSpec{Topic: topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: principal.Bytes()}} +} + +func newSigner(t *testing.T) (crypto.Signer, []byte) { + t.Helper() + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + addr, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + return signer, addr.Bytes() +} + +// peer is the test's end of one stream. +type peer struct { + e *flushEnv + st *flushStream + w protobuf.Writer + signer crypto.Signer + challenge []byte + done chan struct{} // the handler returned +} + +// join attaches a stream declaring owner and returns once the Ack is written. +func (e *flushEnv) join(signer crypto.Signer, owner []byte) *peer { + e.t.Helper() + st := newFlushStream() + p := &peer{e: e, st: st, w: protobuf.NewWriter(st.pw), signer: signer, done: make(chan struct{})} + go func() { + defer close(p.done) + _ = e.svc.handler(context.Background(), p2p.Peer{Address: swarm.RandAddress(e.t)}, st) + }() + go func() { _ = p.w.WriteMsg(&pb.Join{Cohort: e.spec, Owner: owner}) }() + deadline := time.Now().Add(waitFlush) + for { + st.mu.Lock() + var ack pb.Ack + err := protobuf.NewReader(bytes.NewReader(st.out.Bytes())).ReadMsg(&ack) + st.mu.Unlock() + if err == nil { + if ack.Status != pb.Status_OK { + e.t.Fatalf("join: %s", ack.Status) + } + p.challenge = ack.Challenge + return p + } + if time.Now().After(deadline) { + e.t.Fatal("no ack") + } + time.Sleep(5 * time.Millisecond) + } +} + +// send signs a chunk of kind at index under the stream's challenge and writes it. +func (p *peer) send(kind pb.Kind, index uint64, payload []byte) error { + chunkID, err := id(kind, p.e.spec.Topic, p.challenge, index) + if err != nil { + return err + } + ch, err := cac.New(payload) + if err != nil { + return err + } + c, err := soc.New(chunkID, ch).Sign(p.signer) + if err != nil { + return err + } + return p.w.WriteMsg(&pb.Broadcast{Soc: c.Data(), Kind: kind, Challenge: p.challenge, Index: index}) +} + +func (p *peer) mustSend(kind pb.Kind, index uint64, payload []byte) { + p.e.t.Helper() + if err := p.send(kind, index, payload); err != nil { + p.e.t.Fatal(err) + } +} + +func (e *flushEnv) roster(admin *peer, index uint64, addrs ...[]byte) { + e.t.Helper() + payload, err := EncodeRosterMsg(addrs) + if err != nil { + e.t.Fatal(err) + } + admin.mustSend(pb.Kind_ROSTER, index, payload) +} + +func (e *flushEnv) counters() Counters { + e.t.Helper() + key, err := e.spec.Marshal() + if err != nil { + e.t.Fatal(err) + } + e.svc.mtx.Lock() + co := e.svc.cohorts[string(key)] + e.svc.mtx.Unlock() + if co == nil { + e.t.Fatal("no live cohort") + } + return co.counters.snapshot() +} + +const waitFlush = 5 * time.Second + +func waitClosed(t *testing.T, c <-chan struct{}, what string) { + t.Helper() + select { + case <-c: + case <-time.After(waitFlush): + t.Fatalf("%s: timed out", what) + } +} + +func notClosed(t *testing.T, c <-chan struct{}, what string) { + t.Helper() + select { + case <-c: + t.Fatalf("%s: unexpected", what) + default: + } +} + +func eventuallyTrue(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(waitFlush) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not met") + } + time.Sleep(5 * time.Millisecond) + } +} + +// TestFlushDeadline covers the flush deadline of a stream being closed — by +// an EOS or by a revocation — on its write side, where the peer has stopped +// reading and the broker's write blocks, and on its read side, where the peer +// goes on sending or never closes. +func TestFlushDeadline(t *testing.T) { + t.Parallel() + const flush = 300 * time.Millisecond + + t.Run("write: an EOS resets a stream whose peer stopped reading, at the deadline", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + sub.st.stallWrites() + + // the update blocks the subscriber's writer mid-write, outside any flush + admin.mustSend(pb.Kind_DATA, 0, []byte("stuck")) + waitClosed(t, sub.st.stalled, "the write did not block") + + start := time.Now() + admin.mustSend(pb.Kind_EOS, 0, nil) + waitClosed(t, sub.st.reset, "the stalled stream was not reset") + if elapsed := time.Since(start); elapsed < flush*3/4 { + t.Fatalf("reset after %v, before the flush deadline %v", elapsed, flush) + } + waitClosed(t, sub.done, "the stalled stream's handler did not return") + notClosed(t, sub.st.closed, "the stalled stream closed") + + // the sender, which reads, takes its EOS and is closed, not reset + waitClosed(t, admin.st.closed, "the admin's stream was not closed") + notClosed(t, admin.st.reset, "the admin's stream reset") + waitClosed(t, admin.done, "the admin's handler did not return") + fs := admin.st.frames(t) + if len(fs) == 0 || fs[len(fs)-1].Kind != pb.Kind_EOS { + t.Fatalf("the admin's stream did not end on its EOS: %v", fs) + } + }) + + t.Run("write: a revocation resets a publisher stream whose peer stopped reading, at the deadline", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + seatSigner, seatAddr := newSigner(t) + e.roster(admin, 0, seatAddr) + seat := e.join(seatSigner, seatAddr) + eventuallyTrue(t, func() bool { return len(seat.st.frames(t)) > 0 }) // the cue + seat.mustSend(pb.Kind_AUTH, 0, nil) + eventuallyTrue(t, func() bool { + fs := admin.st.frames(t) + return len(fs) > 0 && fs[len(fs)-1].Kind == pb.Kind_ROSTER + }) + seat.st.stallWrites() + + admin.mustSend(pb.Kind_DATA, 0, []byte("stuck")) + waitClosed(t, seat.st.stalled, "the write did not block") + + start := time.Now() + e.roster(admin, 1) + waitClosed(t, seat.st.reset, "the stalled revoked stream was not reset") + if elapsed := time.Since(start); elapsed < flush*3/4 { + t.Fatalf("reset after %v, before the flush deadline %v", elapsed, flush) + } + waitClosed(t, seat.done, "the revoked stream's handler did not return") + notClosed(t, admin.st.reset, "the admin's stream reset") + }) + + t.Run("read: a revoked peer that keeps sending is closed, not reset, its frames dropped", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: 10 * flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + seatSigner, seatAddr := newSigner(t) + e.roster(admin, 0, seatAddr) + seat := e.join(seatSigner, seatAddr) + eventuallyTrue(t, func() bool { return len(seat.st.frames(t)) > 0 }) + seat.mustSend(pb.Kind_AUTH, 0, nil) + eventuallyTrue(t, func() bool { + fs := admin.st.frames(t) + return len(fs) > 0 && fs[len(fs)-1].Kind == pb.Kind_ROSTER + }) + + // the seat floods its stream with updates until it is closed + flooding := make(chan struct{}) + go func() { + defer close(flooding) + for i := uint64(0); ; i++ { + if seat.send(pb.Kind_DATA, i, []byte("flood")) != nil { + return + } + } + }() + eventuallyTrue(t, func() bool { return e.counters().Retransmit == 0 && len(admin.st.frames(t)) > 2 }) + + start := time.Now() + e.roster(admin, 1) + waitClosed(t, seat.st.closed, "the revoked stream was not closed") + if elapsed := time.Since(start); elapsed >= 10*flush { + t.Fatalf("closed after %v: the flood held it to the deadline", elapsed) + } + notClosed(t, seat.st.reset, "the revoked stream reset") + waitClosed(t, seat.done, "the revoked stream's handler did not return") + waitClosed(t, flooding, "the flood did not stop") + + c := e.counters() + if c.WrongStream != 0 || c.InvalidSOC != 0 { + t.Fatalf("a revoked peer's frames punished: %+v", c) + } + fs := seat.st.frames(t) + if last := fs[len(fs)-1]; last.Kind != pb.Kind_ROSTER || last.Index != 1 { + t.Fatalf("the revoked stream did not end on the reduced roster: %s %d", last.Kind, last.Index) + } + }) + + t.Run("read: a silent peer that never closes its side does not hold an ended stream", func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: 10 * flush}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + + start := time.Now() + admin.mustSend(pb.Kind_EOS, 0, nil) + waitClosed(t, sub.st.closed, "the ended stream was not closed") + waitClosed(t, sub.done, "the ended stream's handler did not return") + if elapsed := time.Since(start); elapsed >= 10*flush { + t.Fatalf("closed after %v: the silent peer held it to the deadline", elapsed) + } + notClosed(t, sub.st.reset, "the ended stream reset") + fs := sub.st.frames(t) + if len(fs) != 1 || fs[0].Kind != pb.Kind_EOS { + t.Fatalf("the ended stream got %v, want the EOS alone", fs) + } + }) +} diff --git a/pkg/bps/id.go b/pkg/bps/id.go new file mode 100644 index 00000000000..5d419cc9940 --- /dev/null +++ b/pkg/bps/id.go @@ -0,0 +1,182 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// ChallengeSize is the length of the per-stream challenge issued in Ack. +const ChallengeSize = 32 + +// MaxRosterSize is the most addresses a ROSTER chunk can name. On the wire an +// address takes its 20 bytes, a field tag and a length: 22 bytes, of the 4096 a +// chunk's payload holds. +const MaxRosterSize = swarm.ChunkSize / (1 + 1 + crypto.AddressSize) + +// servicePrefix domain-separates the session feeds of every kind but DATA. +var servicePrefix = []byte("bps-service:v1") + +var ( + // ErrInvalidMessage is a frame whose fields no chunk can make right. + ErrInvalidMessage = errors.New("bps: invalid message") + // ErrInvalidSOC is a chunk that does not have the id derived from its frame, + // or does not validate as its owner's single-owner chunk. + ErrInvalidSOC = errors.New("bps: invalid soc") + // ErrInvalidRoster is a ROSTER chunk whose payload this node will not send: + // one that does not decode as a Roster of 20-byte addresses, or that names + // the admin, or an address twice. + ErrInvalidRoster = errors.New("bps: invalid roster") +) + +// Kind is what a chunk on the wire is. +type Kind int + +const ( + KindUnspecified Kind = iota + KindData // an update of the stream's session feed + // KindAuth is the empty chunk at index 0 of the session's AUTH feed: it authenticates + // a pending stream, or upgrades in place a spectator stream on which a roster naming + // its owner was delivered; it is a heartbeat on a publisher stream, and is never + // delivered. + KindAuth + // KindEOS is the empty chunk at index 0 of the admin's EOS feed: the channel is ended. + KindEOS + // KindRoster is the admin's publisher set as of its index; its payload is a Roster. + KindRoster +) + +func (k Kind) proto() pb.Kind { + switch k { + case KindData: + return pb.Kind_DATA + case KindAuth: + return pb.Kind_AUTH + case KindEOS: + return pb.Kind_EOS + case KindRoster: + return pb.Kind_ROSTER + default: + return pb.Kind_KIND_UNSPECIFIED + } +} + +// SessionTopic returns the topic of the session feed of the given kind: +// keccak256(prefix | topic | challenge), with the prefix empty for DATA and +// "bps-service:v1" | kind for any other kind. +func SessionTopic(kind Kind, topic, challenge []byte) ([]byte, error) { + return sessionTopic(kind.proto(), topic, challenge) +} + +// ID returns the single-owner chunk id of the session feed update at index: +// keccak256(SessionTopic(kind, topic, challenge) | index), index big-endian. +func ID(kind Kind, topic, challenge []byte, index uint64) ([]byte, error) { + return id(kind.proto(), topic, challenge, index) +} + +func sessionTopic(kind pb.Kind, topic, challenge []byte) ([]byte, error) { + var prefix []byte + if kind != pb.Kind_DATA { + prefix = append(append([]byte{}, servicePrefix...), byte(kind)) + } + return crypto.LegacyKeccak256(append(append(prefix, topic...), challenge...)) +} + +func id(kind pb.Kind, topic, challenge []byte, index uint64) ([]byte, error) { + topicS, err := sessionTopic(kind, topic, challenge) + if err != nil { + return nil, err + } + return crypto.LegacyKeccak256(binary.BigEndian.AppendUint64(topicS, index)) +} + +// isServiceKind reports whether the kind is one of the admin's service kinds, +// whose owner is forced to the principal. +func isServiceKind(kind pb.Kind) bool { + return kind == pb.Kind_EOS || kind == pb.Kind_ROSTER +} + +// isEmptyKind reports whether the kind is an empty chunk at index 0 of its feed. +func isEmptyKind(kind pb.Kind) bool { + return kind == pb.Kind_AUTH || kind == pb.Kind_EOS +} + +// parseFrame runs the checks that need no owner — the challenge length, the +// index, the id the frame fields derive, that an AUTH or EOS is empty — and +// returns the frame's chunk, parsed as a single-owner chunk. +func parseFrame(f *pb.Broadcast, topic []byte) (*soc.SOC, error) { + if len(f.Challenge) != ChallengeSize { + return nil, fmt.Errorf("challenge length %d: %w", len(f.Challenge), ErrInvalidMessage) + } + // the last index is one no feed has: the cursor, set to index+1, would overflow + if f.Index == math.MaxUint64 { + return nil, fmt.Errorf("index %d would overflow the feed's cursor: %w", f.Index, ErrInvalidMessage) + } + // a session has one AUTH chunk, and a channel one EOS, at index 0 of its feed + if isEmptyKind(f.Kind) && f.Index != 0 { + return nil, fmt.Errorf("%s chunk at index %d: %w", f.Kind, f.Index, ErrInvalidSOC) + } + want, err := id(f.Kind, topic, f.Challenge, f.Index) + if err != nil { + return nil, err + } + c, err := soc.FromChunk(swarm.NewChunk(swarm.ZeroAddress, f.Soc)) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidSOC, err) + } + if !bytes.Equal(c.ID(), want) { + return nil, fmt.Errorf("id not derived from frame: %w", ErrInvalidSOC) + } + if isEmptyKind(f.Kind) { + // span 0, no payload + if len(f.Soc) != swarm.SocMinChunkSize || binary.LittleEndian.Uint64(c.WrappedChunk().Data()) != 0 { + return nil, fmt.Errorf("%s chunk not empty: %w", f.Kind, ErrInvalidSOC) + } + } + return c, nil +} + +// verify checks the frame as parseFrame does, and that its chunk is owner's: +// the chunk is then valid at keccak256(id | owner). +func verify(f *pb.Broadcast, topic, owner []byte) (*soc.SOC, error) { + c, err := parseFrame(f, topic) + if err != nil { + return nil, err + } + if !bytes.Equal(c.OwnerAddress(), owner) { + return nil, fmt.Errorf("not the owner's soc: %w", ErrInvalidSOC) + } + return c, nil +} + +// payload returns what a single-owner chunk carries, without the span. +func payload(c *soc.SOC) []byte { + return c.WrappedChunk().Data()[swarm.SpanSize:] +} + +// decodeRosterMsg returns the publisher set in a ROSTER chunk's payload, and +// false if it does not decode as a Roster of 20-byte addresses: the admin's +// mistake, which a receiver drops. +func decodeRosterMsg(b []byte) ([][]byte, bool) { + var r pb.Roster + if err := r.Unmarshal(b); err != nil { + return nil, false + } + for _, a := range r.Publishers { + if len(a) != crypto.AddressSize { + return nil, false + } + } + return r.Publishers, true +} diff --git a/pkg/bps/lifecycle_test.go b/pkg/bps/lifecycle_test.go new file mode 100644 index 00000000000..0d07efeee6f --- /dev/null +++ b/pkg/bps/lifecycle_test.go @@ -0,0 +1,198 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// memberOf returns the cohort's member declaring owner. +func (e *flushEnv) memberOf(owner []byte) (*cohort, *member) { + e.t.Helper() + key, err := e.spec.Marshal() + if err != nil { + e.t.Fatal(err) + } + e.svc.mtx.Lock() + co := e.svc.cohorts[string(key)] + e.svc.mtx.Unlock() + if co == nil { + e.t.Fatal("no live cohort") + } + co.mtx.Lock() + defer co.mtx.Unlock() + for m := range co.attached { + if bytes.Equal(m.owner, owner) { + return co, m + } + } + e.t.Fatal("no such member") + return nil, nil +} + +// TestRequestsEndAStreamOnce: any number of requests to close and to reset a +// stream, from any number of goroutines, end it exactly once — closed or +// reset, never both — and never block or panic. +func TestRequestsEndAStreamOnce(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{FlushTimeout: 300 * time.Millisecond}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + co, m := e.memberOf(other) + + var wg sync.WaitGroup + for i := range 64 { + wg.Add(1) + go func() { + defer wg.Done() + if i%2 == 0 { + m.requestClose() + return + } + co.mtx.Lock() + m.requestReset() + co.mtx.Unlock() + }() + } + wg.Wait() + waitClosed(t, sub.done, "the stream's handler did not return") + if resets, closes := sub.st.ended(); resets+closes != 1 { + t.Fatalf("the stream was ended %d times (resets %d, closes %d)", resets+closes, resets, closes) + } + // asking a stream that has ended changes nothing either + m.requestClose() + co.mtx.Lock() + m.requestReset() + co.mtx.Unlock() + notClosed(t, admin.st.reset, "the admin's stream reset") +} + +// TestLeave: when its streams end, a cohort leaves nothing behind — no member, +// no stream count of the peer, no cohort. +func TestLeave(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + + admin.mustSend(pb.Kind_EOS, 0, nil) + waitClosed(t, admin.done, "the admin's handler did not return") + waitClosed(t, sub.done, "the subscriber's handler did not return") + e.svc.mtx.Lock() + defer e.svc.mtx.Unlock() + if len(e.svc.cohorts) != 0 || len(e.svc.peers) != 0 { + t.Fatalf("left behind: %d cohorts, %d peers", len(e.svc.cohorts), len(e.svc.peers)) + } +} + +// TestServiceClose: Close resets every stream and refuses later joins, and a +// second Close does nothing. +func TestServiceClose(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{}) + admin := e.join(e.admin, e.spec.Principal) + admin.mustSend(pb.Kind_AUTH, 0, nil) + _, other := newSigner(t) + sub := e.join(nil, other) + + if err := e.svc.Close(); err != nil { + t.Fatal(err) + } + if err := e.svc.Close(); err != nil { + t.Fatal(err) + } + waitClosed(t, admin.st.reset, "the admin's stream was not reset") + waitClosed(t, sub.st.reset, "the subscriber's stream was not reset") + waitClosed(t, sub.done, "the subscriber's handler did not return") + co, m, _, err := e.svc.join(swarm.RandAddress(t), &pb.Join{Cohort: e.spec, Owner: other}) + if !errors.Is(err, errSessionClosed) || co != nil || m != nil { + t.Fatalf("join after Close: got %v, want %v", err, errSessionClosed) + } +} + +// singleStream is a streamer that opens the one stream the test drives. +type singleStream struct{ st *flushStream } + +func (s singleStream) NewStream(context.Context, swarm.Address, p2p.Headers, string, string, string) (p2p.Stream, error) { + return s.st, nil +} + +// TestSessionEnd: a session ends its stream once — closed after a Close or an +// EOS, reset when the broker goes away — however many Closes are asked for, +// from however many goroutines, before or after, and only then closes Done. +func TestSessionEnd(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + end func(t *testing.T, sess Session, broker *peer) + err error // the session's Err, when it is checked + closed bool // the stream is closed, not reset + }{ + {"a Close closes the stream", func(_ *testing.T, sess Session, _ *peer) { + var wg sync.WaitGroup + for range 64 { + wg.Go(func() { _ = sess.Close() }) + } + wg.Wait() + }, nil, true}, + {"an EOS closes the stream", func(t *testing.T, sess Session, broker *peer) { + t.Helper() + broker.mustSend(pb.Kind_EOS, 0, nil) + if msg := <-sess.Messages(); msg.Kind != KindEOS { + t.Errorf("got %v, want the EOS", msg.Kind) + } + }, ErrEndOfStream, true}, + {"the broker going away resets the stream", func(_ *testing.T, _ Session, broker *peer) { + _ = broker.st.pw.Close() + }, nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newFlushEnv(t, Options{}) + st := newFlushStream() + challenge := make([]byte, ChallengeSize) + // the test is the broker: it answers the Join, and writes the admin's + // frames, signed under a publisher stream's challenge, into the stream + broker := &peer{e: e, st: st, w: protobuf.NewWriter(st.pw), signer: e.admin, challenge: challenge} + go func() { _ = broker.w.WriteMsg(&pb.Ack{Status: pb.Status_OK, Challenge: challenge}) }() + + _, owner := newSigner(t) + client := New(singleStream{st}, nil, false, log.Noop, Options{}) + sess, err := client.Join(context.Background(), JoinRequest{ + Spec: CohortSpec{Topic: e.spec.Topic, Principal: e.spec.Principal}, + Owner: owner, + }) + if err != nil { + t.Fatal(err) + } + tc.end(t, sess, broker) + waitClosed(t, sess.Done(), "Done was not closed") + _ = sess.Close() // asked after the end, it changes nothing + + resets, closes := st.ended() + if resets+closes != 1 || (closes == 1) != tc.closed { + t.Fatalf("resets %d, closes %d; want one, closed %t", resets, closes, tc.closed) + } + if tc.err != nil && !errors.Is(sess.Err(), tc.err) { + t.Fatalf("Err: got %v, want %v", sess.Err(), tc.err) + } + }) + } +} diff --git a/pkg/bps/main_test.go b/pkg/bps/main_test.go new file mode 100644 index 00000000000..784030d903a --- /dev/null +++ b/pkg/bps/main_test.go @@ -0,0 +1,15 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps_test + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/pkg/bps/metrics.go b/pkg/bps/metrics.go new file mode 100644 index 00000000000..2e3fafcf9e8 --- /dev/null +++ b/pkg/bps/metrics.go @@ -0,0 +1,97 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "sync/atomic" + + m "github.com/ethersphere/bee/v2/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +// Counters are the per-cohort counters of the silent outcomes SWIP-74 makes a +// broker expose. +type Counters struct { + WrongStream uint64 // a frame its stream may not send + UnknownKind uint64 // a frame of a kind the broker does not define + WrongChallenge uint64 // a frame whose challenge is not its stream's + InvalidSOC uint64 // a chunk that does not validate at the derived id + AuthTimeout uint64 // a pending stream disconnected at the auth timeout + Retransmit uint64 // a DATA frame below the cursor + QueueReset uint64 // a receiving stream reset for a full queue + InvalidRoster uint64 // a valid ROSTER chunk whose payload does not decode + Revoked uint64 // a frame from a revoked stream, dropped unread + Ended uint64 // a frame from a stream of a cohort an EOS ended, dropped unread +} + +type counters struct { + wrongStream atomic.Uint64 + unknownKind atomic.Uint64 + wrongChallenge atomic.Uint64 + invalidSOC atomic.Uint64 + authTimeout atomic.Uint64 + retransmit atomic.Uint64 + queueReset atomic.Uint64 + invalidRoster atomic.Uint64 + revoked atomic.Uint64 + ended atomic.Uint64 +} + +func (c *counters) snapshot() Counters { + return Counters{ + WrongStream: c.wrongStream.Load(), + UnknownKind: c.unknownKind.Load(), + WrongChallenge: c.wrongChallenge.Load(), + InvalidSOC: c.invalidSOC.Load(), + AuthTimeout: c.authTimeout.Load(), + Retransmit: c.retransmit.Load(), + QueueReset: c.queueReset.Load(), + InvalidRoster: c.invalidRoster.Load(), + Revoked: c.revoked.Load(), + Ended: c.ended.Load(), + } +} + +type metrics struct { + Outcomes *prometheus.CounterVec + Delivered prometheus.Counter + Cohorts prometheus.Gauge +} + +func newMetrics() metrics { + subsystem := "bps" + + return metrics{ + Outcomes: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "dropped_frames", + Help: "Frames dropped or streams reset by the broker, by outcome.", + }, []string{"outcome"}), + Delivered: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "accepted_updates", + Help: "DATA updates accepted by the broker, from every publisher.", + }), + Cohorts: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "cohorts", + Help: "Live cohorts on the broker.", + }), + } +} + +// Metrics returns the prometheus collectors of the service. +func (s *Service) Metrics() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(s.metrics) +} + +// count increments the per-cohort counter c and its prometheus outcome. +func (s *Service) count(c *atomic.Uint64, outcome string) { + c.Add(1) + s.metrics.Outcomes.WithLabelValues(outcome).Inc() +} diff --git a/pkg/bps/multipublisher_test.go b/pkg/bps/multipublisher_test.go new file mode 100644 index 00000000000..20acd4e1c21 --- /dev/null +++ b/pkg/bps/multipublisher_test.go @@ -0,0 +1,771 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps_test + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// seat is a key the admin may roster. +type seat struct { + signer crypto.Signer + addr []byte +} + +func (e *env) newSeat() seat { + e.t.Helper() + key, err := crypto.GenerateSecp256k1Key() + if err != nil { + e.t.Fatal(err) + } + signer := crypto.NewDefaultSigner(key) + addr, err := signer.EthereumAddress() + if err != nil { + e.t.Fatal(err) + } + return seat{signer: signer, addr: addr.Bytes()} +} + +// joinAs joins as the seat's address from a fresh overlay. +func (e *env) joinAs(st seat) (bps.Session, swarm.Address) { + e.t.Helper() + overlay := swarm.RandAddress(e.t) + c, _ := e.client(overlay) + return e.join(c, st.addr), overlay +} + +// grant publishes a roster and waits until the broker has accepted it. +func (e *env) grant(admin bps.Session, index uint64, seats ...seat) { + e.t.Helper() + probe := e.subscribe() + e.roster(admin, index, seats...) + for { + select { + case m := <-probe.Messages(): + if m.Kind == bps.KindRoster && m.Index == index { + return + } + case <-probe.Done(): + e.t.Fatalf("waiting for roster %d: %v", index, probe.Err()) + } + } +} + +// publishAs signs a chunk with the seat's key under the session's challenge +// and publishes it. +func (e *env) publishAs(st seat, s bps.Session, kind bps.Kind, index uint64, payload []byte) []byte { + e.t.Helper() + c := e.chunk(st.signer, kind, s.Challenge(), index, payload) + if err := s.Publish(context.Background(), kind, index, c); err != nil { + e.t.Fatal(err) + } + return c +} + +// roster publishes the admin's roster at index on the admin's publisher stream. +func (e *env) roster(admin bps.Session, index uint64, seats ...seat) []byte { + e.t.Helper() + addrs := make([][]byte, 0, len(seats)) + for _, st := range seats { + addrs = append(addrs, st.addr) + } + payload, err := bps.EncodeRosterMsg(addrs) + if err != nil { + e.t.Fatal(err) + } + return e.publish(admin, bps.KindRoster, index, payload) +} + +func waitGranted(t *testing.T, s bps.Session) { + t.Helper() + select { + case <-s.Granted(): + case <-time.After(waitFor): + t.Fatal("not granted") + } +} + +func expectKind(t *testing.T, s bps.Session, kind bps.Kind, owner []byte, index uint64) bps.Message { + t.Helper() + select { + case m := <-s.Messages(): + if m.Kind != kind || !bytes.Equal(m.Owner, owner) || m.Index != index { + t.Fatalf("got kind %d owner %x index %d, want kind %d owner %x index %d", m.Kind, m.Owner, m.Index, kind, owner, index) + } + return m + case <-s.Done(): + t.Fatalf("stream ended: %v", s.Err()) + case <-time.After(waitFor): + t.Fatalf("timed out waiting for kind %d index %d", kind, index) + } + return bps.Message{} +} + +// drain discards a session's deliveries, so that its stream keeps reading. +func drain(s bps.Session) { + go func() { + for { + select { + case <-s.Messages(): + case <-s.Done(): + return + } + } + }() +} + +func TestGrant(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + sub := e.subscribe() + admin, _ := e.publisher() + + e.roster(admin, 0, st) + m := expectKind(t, sub, bps.KindRoster, e.principal, 0) + if len(m.Roster) != 1 || !bytes.Equal(m.Roster[0], st.addr) { + t.Fatalf("roster %x", m.Roster) + } + // the roster is delivered back to the admin: it admits by it from there + expectKind(t, admin, bps.KindRoster, e.principal, 0) + + // the seat joins as a spectator; the latest roster, naming it, is its cue + seatSess, _ := e.joinAs(st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + c0 := e.publishAs(st, seatSess, bps.KindData, 0, []byte("from the seat")) + + got := expectKind(t, sub, bps.KindData, st.addr, 0) + if !bytes.Equal(got.SOC, c0) { + t.Fatal("wrong chunk") + } + // the admin's publisher stream receives the seat's update; the seat not its own + expectKind(t, admin, bps.KindData, st.addr, 0) + a0 := e.publish(admin, bps.KindData, 0, []byte("from the admin")) + got = expectKind(t, seatSess, bps.KindData, e.principal, 0) + if !bytes.Equal(got.SOC, a0) { + t.Fatal("wrong chunk") + } + expectKind(t, sub, bps.KindData, e.principal, 0) + expectNoMessage(t, admin) +} + +// TestSeatBeforeRoster is the startup order the review found: the seat joins +// before the broker holds any roster, and is upgraded in place by its AUTH +// once the first roster names it. +func TestSeatBeforeRoster(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + sub := e.subscribe() + + seatSess, overlay := e.joinAs(st) + admin, _ := e.publisher() + drain(admin) + e.roster(admin, 0, st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + c := e.publishAs(st, seatSess, bps.KindData, 5, []byte("upgraded in place")) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + got := expectKind(t, sub, bps.KindData, st.addr, 5) + if !bytes.Equal(got.SOC, c) { + t.Fatal("wrong chunk") + } + if e.bl.blocklisted(overlay) || e.counters().WrongStream != 0 { + t.Fatal("an upgrade in place is not a violation") + } +} + +// TestRosteredListens: a rostered address that never authenticates is a +// spectator like any other, and is not disconnected at the auth deadline. +func TestRosteredListens(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{AuthWaitTimeout: 100 * time.Millisecond}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + listener, overlay := e.joinAs(st) + expectKind(t, listener, bps.KindRoster, e.principal, 0) + time.Sleep(300 * time.Millisecond) + e.publish(admin, bps.KindData, 0, []byte("heard")) + expectKind(t, listener, bps.KindData, e.principal, 0) + if e.bl.blocklisted(overlay) || e.counters().AuthTimeout != 0 { + t.Fatal("a listening rostered address is not timed out") + } +} + +func TestUnrosteredAuth(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + _, _ = e.publisher() + overlay := swarm.RandAddress(t) + w, _, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Owner: st.addr}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + auth := e.chunk(st.signer, bps.KindAuth, ack.Challenge, 0, nil) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}); err != nil { + t.Fatal(err) + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) && e.counters().WrongStream == 1 }) +} + +// TestRevokedSpectatorAuth: a spectator sends its AUTH after the roster naming +// it, but a roster revoking it reaches the broker first: the stream is closed, +// nothing is punished. +func TestRevokedSpectatorAuth(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Owner: st.addr}) + var cue pb.Broadcast + if err := r.ReadMsg(&cue); err != nil || cue.Kind != pb.Kind_ROSTER { + t.Fatalf("want the roster as the cue, got %s %v", cue.Kind, err) + } + e.grant(admin, 1) + auth := e.chunk(st.signer, bps.KindAuth, ack.Challenge, 0, nil) + _ = w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}) + eventually(t, func() bool { return e.counters().Revoked == 1 }) + if e.bl.blocklisted(overlay) || e.counters().WrongStream != 0 { + t.Fatal("an AUTH racing its revocation is not punished") + } +} + +func TestRevocation(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + sub := e.subscribe() + e.roster(admin, 0, st) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + + // the seat authenticates on a raw stream, to send after its revocation + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Owner: st.addr}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + var cue pb.Broadcast + if err := r.ReadMsg(&cue); err != nil || cue.Kind != pb.Kind_ROSTER { + t.Fatalf("want the roster as the cue, got %s %v", cue.Kind, err) + } + send := func(kind bps.Kind, index uint64, payload []byte) { + t.Helper() + c := e.chunk(st.signer, kind, ack.Challenge, index, payload) + f := &pb.Broadcast{Kind: pb.Kind(kind), Challenge: ack.Challenge, Index: index, Soc: c} + _ = w.WriteMsg(f) + } + send(bps.KindAuth, 0, nil) + send(bps.KindData, 0, []byte("before")) + expectKind(t, sub, bps.KindData, st.addr, 0) + + // the reduced roster revokes the seat: the stream is written that roster + // and closed; what it sends meanwhile is dropped, unpunished + e.grant(admin, 1) + send(bps.KindData, 1, []byte("in flight")) + m := expectKind(t, sub, bps.KindRoster, e.principal, 1) + if len(m.Roster) != 0 { + t.Fatalf("roster %x", m.Roster) + } + var next pb.Broadcast + if err := r.ReadMsg(&next); err != nil || next.Kind != pb.Kind_ROSTER || next.Index != 1 { + t.Fatalf("want the reduced roster, got %s %d %v", next.Kind, next.Index, err) + } + if err := r.ReadMsg(&next); err == nil { + t.Fatalf("want the stream closed, got %s", next.Kind) + } + expectNoMessage(t, sub) + if e.bl.blocklisted(overlay) || e.counters().WrongStream != 0 { + t.Fatal("a revoked peer is not punished") + } + if pos := sub.Position(); len(pos.Roster) != 0 || pos.RosterCursor != 2 { + t.Fatalf("position %+v", pos) + } + + // rejoining, the seat is a spectator that may not authenticate + spectator, _ := e.joinAs(st) + expectKind(t, spectator, bps.KindRoster, e.principal, 1) + select { + case <-spectator.Granted(): + t.Fatal("a revoked owner is not granted") + default: + } +} + +// TestRevokeInFlight: the admin revokes a seat whose update the broker accepted +// just before the roster; the admin's own session, admitting by the echoed +// roster, takes the update and is not reset. +func TestRevokeInFlight(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + e.roster(admin, 0, st) + expectKind(t, admin, bps.KindRoster, e.principal, 0) + seatSess, _ := e.joinAs(st) + drain(seatSess) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + e.publishAs(st, seatSess, bps.KindData, 0, []byte("last")) + time.Sleep(50 * time.Millisecond) // the broker accepts the update first + e.roster(admin, 1) + expectKind(t, admin, bps.KindData, st.addr, 0) + expectKind(t, admin, bps.KindRoster, e.principal, 1) + select { + case <-admin.Done(): + t.Fatalf("the admin's stream ended: %v", admin.Err()) + default: + } +} + +func TestRevokedSession(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + seatSess, _ := e.joinAs(st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + + e.roster(admin, 1) + expectKind(t, seatSess, bps.KindRoster, e.principal, 1) + expectDone(t, seatSess) + if err := seatSess.Publish(context.Background(), bps.KindData, 0, nil); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("got %v, want %v", err, bps.ErrNotPublisher) + } +} + +func TestCursorsPerPublisher(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + sub := e.subscribe() + e.roster(admin, 0, st) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + seatSess, _ := e.joinAs(st) + drain(seatSess) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + + // each publisher has its own cursor + e.publish(admin, bps.KindData, 3, []byte("a3")) + expectKind(t, sub, bps.KindData, e.principal, 3) + e.publishAs(st, seatSess, bps.KindData, 0, []byte("s0")) + expectKind(t, sub, bps.KindData, st.addr, 0) + // below its own cursor: a retransmit + e.publishAs(st, seatSess, bps.KindData, 0, []byte("s0")) + eventually(t, func() bool { return e.counters().Retransmit == 1 }) + expectNoMessage(t, sub) + + pos := sub.Position() + if pos.Cursors[string(e.principal)] != 4 || pos.Cursors[string(st.addr)] != 1 { + t.Fatalf("cursors %v", pos.Cursors) + } +} + +func TestRosterFeed(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + sub := e.subscribe() + + e.roster(admin, 0, st) + expectKind(t, sub, bps.KindRoster, e.principal, 0) + // a roster below the roster cursor is a retransmit + e.roster(admin, 0, st) + eventually(t, func() bool { return e.counters().Retransmit == 1 }) + + // a valid chunk whose payload is not a roster changes nothing; a session + // refuses to send one, so it comes from a raw stream of the admin's + w, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Owner: e.principal}) + auth := e.chunk(e.signer, bps.KindAuth, ack.Challenge, 0, nil) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}); err != nil { + t.Fatal(err) + } + bad := e.chunk(e.signer, bps.KindRoster, ack.Challenge, 1, []byte{0x0a, 0x02, 0x01, 0x02}) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_ROSTER, Challenge: ack.Challenge, Index: 1, Soc: bad}); err != nil { + t.Fatal(err) + } + eventually(t, func() bool { return e.counters().InvalidRoster == 1 }) + // nor does one with an empty entry, which would name a stream with no owner + empty := e.chunk(e.signer, bps.KindRoster, ack.Challenge, 1, []byte{0x0a, 0x00}) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_ROSTER, Challenge: ack.Challenge, Index: 1, Soc: empty}); err != nil { + t.Fatal(err) + } + eventually(t, func() bool { return e.counters().InvalidRoster == 2 }) + e.roster(admin, 1) + expectKind(t, sub, bps.KindRoster, e.principal, 1) + + // a late joiner receives the latest roster first + late := e.subscribe() + m := expectKind(t, late, bps.KindRoster, e.principal, 1) + if len(m.Roster) != 0 { + t.Fatalf("roster %x", m.Roster) + } +} + +func TestServiceKindsAdminOnly(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + overlay := swarm.RandAddress(t) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Owner: st.addr}) + var cue pb.Broadcast + if err := r.ReadMsg(&cue); err != nil { + t.Fatal(err) + } + auth := e.chunk(st.signer, bps.KindAuth, ack.Challenge, 0, nil) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}); err != nil { + t.Fatal(err) + } + payload, _ := bps.EncodeRosterMsg([][]byte{st.addr}) + ros := e.chunk(st.signer, bps.KindRoster, ack.Challenge, 1, payload) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_ROSTER, Challenge: ack.Challenge, Index: 1, Soc: ros}); err != nil { + t.Fatal(err) + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) && e.counters().WrongStream == 1 }) +} + +func TestEndOfStream(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + admin, _ := e.publisher() + sub := e.subscribe() + second, _ := e.pending() + e.publish(admin, bps.KindData, 0, []byte("last words")) + expectKind(t, sub, bps.KindData, e.principal, 0) + + e.publish(admin, bps.KindEOS, 0, nil) + // every stream is told — the sender and a pending one included — and closed + for _, s := range []bps.Session{sub, admin, second} { + expectKind(t, s, bps.KindEOS, e.principal, 0) + expectDone(t, s) + if !errors.Is(s.Err(), bps.ErrEndOfStream) { + t.Fatalf("got %v, want %v", s.Err(), bps.ErrEndOfStream) + } + } + eventually(t, func() bool { + _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}) + return !ok + }) + + // a channel the subscriber has seen end is not rejoined + pos := sub.Position() + if !pos.Ended { + t.Fatal("not ended") + } + c, _ := e.client(swarm.RandAddress(t)) + addr := make([]byte, 20) + _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Owner: addr, Position: pos}) + if !errors.Is(err, bps.ErrChannelEnded) { + t.Fatalf("got %v, want %v", err, bps.ErrChannelEnded) + } + // a Join that does not know creates a fresh cohort + fresh := e.subscribe() + expectNoMessage(t, fresh) + if _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}); !ok { + t.Fatal("no fresh cohort") + } +} + +func TestPublisherStreamsBound(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxPublisherStreams: 2}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + sess := make([]bps.Session, 0, 3) + for range 3 { + s, _ := e.joinAs(st) + drain(s) + waitGranted(t, s) + e.publishAs(st, s, bps.KindAuth, 0, nil) + sess = append(sess, s) + time.Sleep(50 * time.Millisecond) + } + // the third auth resets the oldest + expectDone(t, sess[0]) + select { + case <-sess[1].Done(): + t.Fatal("second stream reset") + case <-sess[2].Done(): + t.Fatal("third stream reset") + default: + } +} + +// TestRosteredPastFullSet: a rostered owner that finds the fan-out set full +// is pending — the roster, nothing else — and authenticates as any grantee; +// one that does not is reset at the auth deadline, not blocklisted. +func TestRosteredPastFullSet(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxSubsPerCohort: 1, AuthWaitTimeout: 300 * time.Millisecond}) + st, idle := e.newSeat(), e.newSeat() + admin, _ := e.publisher() + drain(admin) + probe := e.subscribe() // fills the fan-out set + e.roster(admin, 0, st, idle) + expectKind(t, probe, bps.KindRoster, e.principal, 0) + + seatSess, _ := e.joinAs(st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + e.publishAs(st, seatSess, bps.KindData, 0, []byte("past the bound")) + expectKind(t, probe, bps.KindData, st.addr, 0) + + listener, overlay := e.joinAs(idle) + expectKind(t, listener, bps.KindRoster, e.principal, 0) + e.publish(admin, bps.KindData, 0, []byte("not for a pending stream")) + expectKind(t, probe, bps.KindData, e.principal, 0) + expectDone(t, listener) + if e.bl.blocklisted(overlay) || e.counters().AuthTimeout != 1 { + t.Fatal("a rostered pending stream times out without a blocklist") + } +} + +// TestSeatPublishChecks: a session refuses what its seat may not send yet, or at all. +func TestSeatPublishChecks(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + seatSess, overlay := e.joinAs(st) + publish := func(kind bps.Kind, payload []byte) error { + t.Helper() + return seatSess.Publish(context.Background(), kind, 0, e.chunk(st.signer, kind, seatSess.Challenge(), 0, payload)) + } + + if err := publish(bps.KindAuth, nil); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("auth before the grant: got %v, want %v", err, bps.ErrNotPublisher) + } + + admin, _ := e.publisher() + drain(admin) + e.roster(admin, 0, st) + expectKind(t, seatSess, bps.KindRoster, e.principal, 0) + waitGranted(t, seatSess) + + if err := publish(bps.KindData, []byte("x")); !errors.Is(err, bps.ErrInvalidMessage) { + t.Fatalf("data before the auth: got %v, want %v", err, bps.ErrInvalidMessage) + } + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) + if err := publish(bps.KindRoster, nil); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("roster from a seat: got %v, want %v", err, bps.ErrNotPublisher) + } + if err := publish(bps.KindEOS, nil); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("eos from a seat: got %v, want %v", err, bps.ErrNotPublisher) + } + // none of it reached the broker + if e.bl.blocklisted(overlay) || e.counters().WrongStream != 0 { + t.Fatal("a refused frame reached the broker") + } +} + +// TestGrantOnReconnect: a seat that passes back its position does not get the +// broker's re-sent roster again, which is below its roster cursor, but it is the +// cue on this stream all the same. So a dApp cannot go by the ROSTER delivery +// alone: Granted is what tells it. +func TestGrantOnReconnect(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + c, _ := e.client(swarm.RandAddress(t)) + seatSess, err := c.Join(context.Background(), bps.JoinRequest{ + Broker: e.brokerAddr, + Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, + Owner: st.addr, + Position: bps.Position{Cursors: map[string]uint64{}, Roster: [][]byte{st.addr}, RosterCursor: 1}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = seatSess.Close() }) + waitGranted(t, seatSess) + expectNoMessage(t, seatSess) + e.publishAs(st, seatSess, bps.KindAuth, 0, nil) +} + +// TestRosterNamingTheAdmin: a session refuses to send one (see TestPublishChecks), +// but a broker and a subscriber take a roster that names the admin, or an address +// twice, as it comes, and it changes nothing: the admin stays a publisher, by that +// roster and by a later one that does not name it, and the rest of it holds. +func TestRosterNamingTheAdmin(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + roster func(admin, seat []byte) [][]byte + }{ + {"the admin alone", func(admin, _ []byte) [][]byte { return [][]byte{admin} }}, + {"the admin with another", func(admin, seat []byte) [][]byte { return [][]byte{admin, seat} }}, + {"an address twice", func(_, seat []byte) [][]byte { return [][]byte{seat, seat} }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + sub := e.subscribe() + w, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Owner: e.principal}) + write := func(f *pb.Broadcast) { + t.Helper() + f.Challenge = ack.Challenge + if err := w.WriteMsg(f); err != nil { + t.Fatal(err) + } + } + roster := func(index uint64, addrs [][]byte) { + t.Helper() + payload, err := bps.EncodeRosterMsg(addrs) + if err != nil { + t.Fatal(err) + } + write(&pb.Broadcast{Kind: pb.Kind_ROSTER, Index: index, Soc: e.chunk(e.signer, bps.KindRoster, ack.Challenge, index, payload)}) + expectKind(t, sub, bps.KindRoster, e.principal, index) + } + data := func(index uint64) { + t.Helper() + write(&pb.Broadcast{Kind: pb.Kind_DATA, Index: index, Soc: e.chunk(e.signer, bps.KindData, ack.Challenge, index, []byte("x"))}) + expectKind(t, sub, bps.KindData, e.principal, index) + } + write(&pb.Broadcast{Kind: pb.Kind_AUTH, Soc: e.chunk(e.signer, bps.KindAuth, ack.Challenge, 0, nil)}) + + roster(0, tc.roster(e.principal, st.addr)) + data(0) + roster(1, nil) // names no one: the admin is a publisher all the same + data(1) + if got := e.counters(); got.InvalidRoster != 0 || got.Revoked != 0 || got.WrongStream != 0 || got.InvalidSOC != 0 { + t.Fatalf("counters %+v", got) + } + }) + } +} + +// TestNoOwner: a stream may declare no owner. It listens like any spectator, within +// the fan-out bound, and can never upgrade, even if the key behind it is on the +// roster: no roster can name it. +func TestNoOwner(t *testing.T) { + t.Parallel() + + t.Run("it listens, and its session refuses to publish", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + c, _ := e.client(swarm.RandAddress(t)) + listener := e.join(c, nil) + expectKind(t, listener, bps.KindRoster, e.principal, 0) + e.publish(admin, bps.KindData, 0, []byte("heard")) + expectKind(t, listener, bps.KindData, e.principal, 0) + + select { + case <-listener.Granted(): + t.Fatal("a stream with no owner is not granted") + default: + } + auth := e.chunk(st.signer, bps.KindAuth, listener.Challenge(), 0, nil) + if err := listener.Publish(context.Background(), bps.KindAuth, 0, auth); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("got %v, want %v", err, bps.ErrNotPublisher) + } + + // a later roster changes nothing for it + e.grant(admin, 1) + expectKind(t, listener, bps.KindRoster, e.principal, 1) + e.publish(admin, bps.KindData, 1, []byte("still heard")) + expectKind(t, listener, bps.KindData, e.principal, 1) + }) + + t.Run("at the broker, an AUTH on it is not what the stream may send", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + st := e.newSeat() + admin, _ := e.publisher() + drain(admin) + e.grant(admin, 0, st) + + overlay := swarm.RandAddress(t) + w, _, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec()}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + auth := e.chunk(st.signer, bps.KindAuth, ack.Challenge, 0, nil) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_AUTH, Challenge: ack.Challenge, Soc: auth}); err != nil { + t.Fatal(err) + } + eventually(t, func() bool { return e.bl.blocklisted(overlay) && e.counters().WrongStream == 1 }) + }) + + t.Run("it counts against the fan-out bound", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxSubsPerCohort: 1}) + e.subscribe() + if _, _, ack := e.raw(&pb.Join{Cohort: e.spec()}); ack.Status != pb.Status_FULL { + t.Fatalf("got %s, want FULL", ack.Status) + } + }) +} + +// TestMaxRoster: the most addresses a roster can name is what fits in a chunk. +func TestMaxRoster(t *testing.T) { + t.Parallel() + addrs := make([][]byte, bps.MaxRosterSize+1) + for i := range addrs { + addrs[i] = make([]byte, crypto.AddressSize) + } + fits, err := bps.EncodeRosterMsg(addrs[:bps.MaxRosterSize]) + if err != nil { + t.Fatal(err) + } + if _, err := cac.New(fits); err != nil { + t.Fatalf("%d addresses, %d bytes: %v", bps.MaxRosterSize, len(fits), err) + } + over, err := bps.EncodeRosterMsg(addrs) + if err != nil { + t.Fatal(err) + } + if _, err := cac.New(over); err == nil { + t.Fatalf("%d addresses, %d bytes fit in a chunk", len(addrs), len(over)) + } +} diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go new file mode 100644 index 00000000000..98350a87c81 --- /dev/null +++ b/pkg/bps/pb/bps.pb.go @@ -0,0 +1,1449 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: bps.proto + +package pb + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// What the topic binds to. One binding. +type TopicBinding int32 + +const ( + TopicBinding_TOPIC_BINDING_UNSPECIFIED TopicBinding = 0 + TopicBinding_FEED_TOPIC TopicBinding = 1 +) + +var TopicBinding_name = map[int32]string{ + 0: "TOPIC_BINDING_UNSPECIFIED", + 1: "FEED_TOPIC", +} + +var TopicBinding_value = map[string]int32{ + "TOPIC_BINDING_UNSPECIFIED": 0, + "FEED_TOPIC": 1, +} + +func (x TopicBinding) String() string { + return proto.EnumName(TopicBinding_name, int32(x)) +} + +func (TopicBinding) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{0} +} + +type Status int32 + +const ( + Status_STATUS_UNSPECIFIED Status = 0 + Status_OK Status = 1 + Status_FULL Status = 2 + Status_REJECTED Status = 3 +) + +var Status_name = map[int32]string{ + 0: "STATUS_UNSPECIFIED", + 1: "OK", + 2: "FULL", + 3: "REJECTED", +} + +var Status_value = map[string]int32{ + "STATUS_UNSPECIFIED": 0, + "OK": 1, + "FULL": 2, + "REJECTED": 3, +} + +func (x Status) String() string { + return proto.EnumName(Status_name, int32(x)) +} + +func (Status) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{1} +} + +// What a chunk on this wire is. +type Kind int32 + +const ( + Kind_KIND_UNSPECIFIED Kind = 0 + Kind_DATA Kind = 1 + Kind_AUTH Kind = 2 + // a pending stream, or a spectator stream a delivered roster names; + // a heartbeat on a publisher stream; never delivered + Kind_EOS Kind = 3 + Kind_ROSTER Kind = 4 +) + +var Kind_name = map[int32]string{ + 0: "KIND_UNSPECIFIED", + 1: "DATA", + 2: "AUTH", + 3: "EOS", + 4: "ROSTER", +} + +var Kind_value = map[string]int32{ + "KIND_UNSPECIFIED": 0, + "DATA": 1, + "AUTH": 2, + "EOS": 3, + "ROSTER": 4, +} + +func (x Kind) String() string { + return proto.EnumName(Kind_name, int32(x)) +} + +func (Kind) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{2} +} + +// The cohort's identity, keyed by its canonical serialisation. +type CohortSpec struct { + Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` + Principal []byte `protobuf:"bytes,3,opt,name=principal,proto3" json:"principal,omitempty"` +} + +func (m *CohortSpec) Reset() { *m = CohortSpec{} } +func (m *CohortSpec) String() string { return proto.CompactTextString(m) } +func (*CohortSpec) ProtoMessage() {} +func (*CohortSpec) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{0} +} +func (m *CohortSpec) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CohortSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CohortSpec.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CohortSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_CohortSpec.Merge(m, src) +} +func (m *CohortSpec) XXX_Size() int { + return m.Size() +} +func (m *CohortSpec) XXX_DiscardUnknown() { + xxx_messageInfo_CohortSpec.DiscardUnknown(m) +} + +var xxx_messageInfo_CohortSpec proto.InternalMessageInfo + +func (m *CohortSpec) GetTopic() []byte { + if m != nil { + return m.Topic + } + return nil +} + +func (m *CohortSpec) GetBinding() TopicBinding { + if m != nil { + return m.Binding + } + return TopicBinding_TOPIC_BINDING_UNSPECIFIED +} + +func (m *CohortSpec) GetPrincipal() []byte { + if m != nil { + return m.Principal + } + return nil +} + +// Peer -> broker: the first frame on a fresh stream. +type Join struct { + Cohort *CohortSpec `protobuf:"bytes,1,opt,name=cohort,proto3" json:"cohort,omitempty"` + Owner []byte `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` +} + +func (m *Join) Reset() { *m = Join{} } +func (m *Join) String() string { return proto.CompactTextString(m) } +func (*Join) ProtoMessage() {} +func (*Join) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{1} +} +func (m *Join) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Join) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Join.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Join) XXX_Merge(src proto.Message) { + xxx_messageInfo_Join.Merge(m, src) +} +func (m *Join) XXX_Size() int { + return m.Size() +} +func (m *Join) XXX_DiscardUnknown() { + xxx_messageInfo_Join.DiscardUnknown(m) +} + +var xxx_messageInfo_Join proto.InternalMessageInfo + +func (m *Join) GetCohort() *CohortSpec { + if m != nil { + return m.Cohort + } + return nil +} + +func (m *Join) GetOwner() []byte { + if m != nil { + return m.Owner + } + return nil +} + +// Broker -> peer, answering Join. A non-OK Ack ends the stream. +type Ack struct { + Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` + Challenge []byte `protobuf:"bytes,2,opt,name=challenge,proto3" json:"challenge,omitempty"` +} + +func (m *Ack) Reset() { *m = Ack{} } +func (m *Ack) String() string { return proto.CompactTextString(m) } +func (*Ack) ProtoMessage() {} +func (*Ack) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{2} +} +func (m *Ack) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Ack) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Ack.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Ack) XXX_Merge(src proto.Message) { + xxx_messageInfo_Ack.Merge(m, src) +} +func (m *Ack) XXX_Size() int { + return m.Size() +} +func (m *Ack) XXX_DiscardUnknown() { + xxx_messageInfo_Ack.DiscardUnknown(m) +} + +var xxx_messageInfo_Ack proto.InternalMessageInfo + +func (m *Ack) GetStatus() Status { + if m != nil { + return m.Status + } + return Status_STATUS_UNSPECIFIED +} + +func (m *Ack) GetChallenge() []byte { + if m != nil { + return m.Challenge + } + return nil +} + +// The payload of a ROSTER chunk. +type Roster struct { + Publishers [][]byte `protobuf:"bytes,1,rep,name=publishers,proto3" json:"publishers,omitempty"` +} + +func (m *Roster) Reset() { *m = Roster{} } +func (m *Roster) String() string { return proto.CompactTextString(m) } +func (*Roster) ProtoMessage() {} +func (*Roster) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{3} +} +func (m *Roster) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Roster) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Roster.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Roster) XXX_Merge(src proto.Message) { + xxx_messageInfo_Roster.Merge(m, src) +} +func (m *Roster) XXX_Size() int { + return m.Size() +} +func (m *Roster) XXX_DiscardUnknown() { + xxx_messageInfo_Roster.DiscardUnknown(m) +} + +var xxx_messageInfo_Roster proto.InternalMessageInfo + +func (m *Roster) GetPublishers() [][]byte { + if m != nil { + return m.Publishers + } + return nil +} + +// Every frame after the handshake. soc is the index-th update of the session feed: +// a single-owner chunk whose id is derived from kind, challenge and index (see bps.ID). +type Broadcast struct { + Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` + Kind Kind `protobuf:"varint,2,opt,name=kind,proto3,enum=bps.Kind" json:"kind,omitempty"` + Challenge []byte `protobuf:"bytes,3,opt,name=challenge,proto3" json:"challenge,omitempty"` + Index uint64 `protobuf:"varint,4,opt,name=index,proto3" json:"index,omitempty"` +} + +func (m *Broadcast) Reset() { *m = Broadcast{} } +func (m *Broadcast) String() string { return proto.CompactTextString(m) } +func (*Broadcast) ProtoMessage() {} +func (*Broadcast) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{4} +} +func (m *Broadcast) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Broadcast.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Broadcast) XXX_Merge(src proto.Message) { + xxx_messageInfo_Broadcast.Merge(m, src) +} +func (m *Broadcast) XXX_Size() int { + return m.Size() +} +func (m *Broadcast) XXX_DiscardUnknown() { + xxx_messageInfo_Broadcast.DiscardUnknown(m) +} + +var xxx_messageInfo_Broadcast proto.InternalMessageInfo + +func (m *Broadcast) GetSoc() []byte { + if m != nil { + return m.Soc + } + return nil +} + +func (m *Broadcast) GetKind() Kind { + if m != nil { + return m.Kind + } + return Kind_KIND_UNSPECIFIED +} + +func (m *Broadcast) GetChallenge() []byte { + if m != nil { + return m.Challenge + } + return nil +} + +func (m *Broadcast) GetIndex() uint64 { + if m != nil { + return m.Index + } + return 0 +} + +func init() { + proto.RegisterEnum("bps.TopicBinding", TopicBinding_name, TopicBinding_value) + proto.RegisterEnum("bps.Status", Status_name, Status_value) + proto.RegisterEnum("bps.Kind", Kind_name, Kind_value) + proto.RegisterType((*CohortSpec)(nil), "bps.CohortSpec") + proto.RegisterType((*Join)(nil), "bps.Join") + proto.RegisterType((*Ack)(nil), "bps.Ack") + proto.RegisterType((*Roster)(nil), "bps.Roster") + proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") +} + +func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } + +var fileDescriptor_786299925f8760d5 = []byte{ + // 458 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x52, 0x4d, 0x6b, 0xdb, 0x40, + 0x10, 0xb5, 0x3e, 0xaa, 0xc4, 0x13, 0xe1, 0x6e, 0x97, 0x50, 0x5c, 0x48, 0x84, 0x51, 0x0f, 0x15, + 0x2e, 0xe4, 0xe0, 0x9e, 0x0b, 0x95, 0xad, 0x75, 0xa3, 0x38, 0xd8, 0x61, 0x25, 0x5f, 0x7a, 0x31, + 0xfa, 0x22, 0x5e, 0xe2, 0xee, 0x2e, 0x92, 0x42, 0xfb, 0x33, 0xfa, 0xb3, 0x7a, 0xcc, 0xb1, 0xc7, + 0x62, 0xff, 0x91, 0xa2, 0x95, 0x83, 0x53, 0xdf, 0x76, 0xde, 0x9b, 0x79, 0xf3, 0xe6, 0xb1, 0xd0, + 0x4d, 0x65, 0x75, 0x25, 0x4b, 0x51, 0x0b, 0x6c, 0xa4, 0xb2, 0x72, 0xbf, 0x03, 0x4c, 0xc4, 0x5a, + 0x94, 0x75, 0x24, 0x8b, 0x0c, 0x9f, 0xc3, 0xab, 0x5a, 0x48, 0x96, 0xf5, 0xb5, 0x81, 0xe6, 0xd9, + 0xb4, 0x2d, 0xf0, 0x47, 0x38, 0x49, 0x19, 0xcf, 0x19, 0xbf, 0xef, 0xeb, 0x03, 0xcd, 0xeb, 0x8d, + 0xde, 0x5c, 0x35, 0x2a, 0x71, 0x43, 0x8e, 0x5b, 0x82, 0x3e, 0x77, 0xe0, 0x0b, 0xe8, 0xca, 0x92, + 0xf1, 0x8c, 0xc9, 0x64, 0xd3, 0x37, 0x94, 0xcc, 0x01, 0x70, 0x09, 0x98, 0x37, 0x82, 0x71, 0xfc, + 0x01, 0xac, 0x4c, 0xad, 0x55, 0x9b, 0xce, 0x46, 0xaf, 0x95, 0xe2, 0xc1, 0x09, 0xdd, 0xd3, 0x8d, + 0x23, 0xf1, 0x83, 0x17, 0xa5, 0xda, 0x6c, 0xd3, 0xb6, 0x70, 0xaf, 0xc1, 0xf0, 0xb3, 0x07, 0xfc, + 0x1e, 0xac, 0xaa, 0x4e, 0xea, 0xc7, 0x4a, 0xa9, 0xf4, 0x46, 0x67, 0x4a, 0x25, 0x52, 0x10, 0xdd, + 0x53, 0x8d, 0xa1, 0x6c, 0x9d, 0x6c, 0x36, 0x05, 0xbf, 0x2f, 0xf6, 0x2a, 0x07, 0xc0, 0xf5, 0xc0, + 0xa2, 0xa2, 0xaa, 0x8b, 0x12, 0x3b, 0x00, 0xf2, 0x31, 0xdd, 0xb0, 0x6a, 0x5d, 0x94, 0x8d, 0xa0, + 0xe1, 0xd9, 0xf4, 0x05, 0xe2, 0x4a, 0xe8, 0x8e, 0x4b, 0x91, 0xe4, 0x59, 0x52, 0xd5, 0x18, 0x81, + 0x51, 0x89, 0xe7, 0x98, 0x9a, 0x27, 0xbe, 0x04, 0xf3, 0x81, 0xf1, 0x7c, 0x9f, 0x50, 0x57, 0x39, + 0x99, 0x31, 0x9e, 0x53, 0x05, 0xff, 0xef, 0xc2, 0x38, 0x72, 0xd1, 0x5c, 0xc9, 0x78, 0x5e, 0xfc, + 0xec, 0x9b, 0x03, 0xcd, 0x33, 0x69, 0x5b, 0x0c, 0x3f, 0x83, 0xfd, 0x32, 0x63, 0x7c, 0x09, 0xef, + 0xe2, 0xc5, 0x5d, 0x38, 0x59, 0x8d, 0xc3, 0x79, 0x10, 0xce, 0xbf, 0xae, 0x96, 0xf3, 0xe8, 0x8e, + 0x4c, 0xc2, 0x69, 0x48, 0x02, 0xd4, 0xc1, 0x3d, 0x80, 0x29, 0x21, 0xc1, 0x4a, 0xf5, 0x20, 0x6d, + 0xf8, 0x05, 0xac, 0x36, 0x0a, 0xfc, 0x16, 0x70, 0x14, 0xfb, 0xf1, 0x32, 0x3a, 0x9a, 0xb0, 0x40, + 0x5f, 0xcc, 0x90, 0x86, 0x4f, 0xc1, 0x9c, 0x2e, 0x6f, 0x6f, 0x91, 0x8e, 0x6d, 0x38, 0xa5, 0xe4, + 0x86, 0x4c, 0x62, 0x12, 0x20, 0x63, 0x48, 0xc0, 0x6c, 0x4e, 0xc0, 0xe7, 0x80, 0x66, 0xe1, 0x3c, + 0x38, 0x9a, 0x3e, 0x05, 0x33, 0xf0, 0x63, 0xbf, 0x9d, 0xf7, 0x97, 0xf1, 0x35, 0xd2, 0xf1, 0x09, + 0x18, 0x64, 0x11, 0x21, 0x03, 0x03, 0x58, 0x74, 0x11, 0xc5, 0x84, 0x22, 0x73, 0x7c, 0xf1, 0x7b, + 0xeb, 0x68, 0x4f, 0x5b, 0x47, 0xfb, 0xbb, 0x75, 0xb4, 0x5f, 0x3b, 0xa7, 0xf3, 0xb4, 0x73, 0x3a, + 0x7f, 0x76, 0x4e, 0xe7, 0x9b, 0x2e, 0xd3, 0xd4, 0x52, 0xbf, 0xf1, 0xd3, 0xbf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xde, 0x3e, 0x41, 0xd4, 0x9a, 0x02, 0x00, 0x00, +} + +func (m *CohortSpec) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CohortSpec) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CohortSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Principal) > 0 { + i -= len(m.Principal) + copy(dAtA[i:], m.Principal) + i = encodeVarintBps(dAtA, i, uint64(len(m.Principal))) + i-- + dAtA[i] = 0x1a + } + if m.Binding != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x10 + } + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Join) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Join) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Owner) > 0 { + i -= len(m.Owner) + copy(dAtA[i:], m.Owner) + i = encodeVarintBps(dAtA, i, uint64(len(m.Owner))) + i-- + dAtA[i] = 0x12 + } + if m.Cohort != nil { + { + size, err := m.Cohort.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBps(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Ack) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Ack) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Challenge) > 0 { + i -= len(m.Challenge) + copy(dAtA[i:], m.Challenge) + i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) + i-- + dAtA[i] = 0x12 + } + if m.Status != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Status)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Roster) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Roster) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Roster) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Publishers) > 0 { + for iNdEx := len(m.Publishers) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Publishers[iNdEx]) + copy(dAtA[i:], m.Publishers[iNdEx]) + i = encodeVarintBps(dAtA, i, uint64(len(m.Publishers[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *Broadcast) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Broadcast) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Index != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Index)) + i-- + dAtA[i] = 0x20 + } + if len(m.Challenge) > 0 { + i -= len(m.Challenge) + copy(dAtA[i:], m.Challenge) + i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) + i-- + dAtA[i] = 0x1a + } + if m.Kind != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Kind)) + i-- + dAtA[i] = 0x10 + } + if len(m.Soc) > 0 { + i -= len(m.Soc) + copy(dAtA[i:], m.Soc) + i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintBps(dAtA []byte, offset int, v uint64) int { + offset -= sovBps(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *CohortSpec) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Topic) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Binding != 0 { + n += 1 + sovBps(uint64(m.Binding)) + } + l = len(m.Principal) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *Join) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Cohort != nil { + l = m.Cohort.Size() + n += 1 + l + sovBps(uint64(l)) + } + l = len(m.Owner) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *Ack) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != 0 { + n += 1 + sovBps(uint64(m.Status)) + } + l = len(m.Challenge) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + return n +} + +func (m *Roster) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Publishers) > 0 { + for _, b := range m.Publishers { + l = len(b) + n += 1 + l + sovBps(uint64(l)) + } + } + return n +} + +func (m *Broadcast) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Soc) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Kind != 0 { + n += 1 + sovBps(uint64(m.Kind)) + } + l = len(m.Challenge) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Index != 0 { + n += 1 + sovBps(uint64(m.Index)) + } + return n +} + +func sovBps(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozBps(x uint64) (n int) { + return sovBps(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *CohortSpec) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CohortSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CohortSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) + if m.Topic == nil { + m.Topic = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= TopicBinding(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Principal", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Principal = append(m.Principal[:0], dAtA[iNdEx:postIndex]...) + if m.Principal == nil { + m.Principal = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Join) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Join: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Join: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Cohort", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Cohort == nil { + m.Cohort = &CohortSpec{} + } + if err := m.Cohort.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Owner", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Owner = append(m.Owner[:0], dAtA[iNdEx:postIndex]...) + if m.Owner == nil { + m.Owner = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Ack) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Ack: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Ack: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= Status(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Roster) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Roster: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Roster: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Publishers", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Publishers = append(m.Publishers, make([]byte, postIndex-iNdEx)) + copy(m.Publishers[len(m.Publishers)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Broadcast) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Broadcast: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Broadcast: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) + if m.Soc == nil { + m.Soc = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType) + } + m.Kind = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Kind |= Kind(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + m.Index = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Index |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipBps(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBps + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBps + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBps + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthBps + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupBps + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthBps + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthBps = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowBps = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupBps = fmt.Errorf("proto: unexpected end of group") +) diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto new file mode 100644 index 00000000000..3bfb975a505 --- /dev/null +++ b/pkg/bps/pb/bps.proto @@ -0,0 +1,71 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Wire format of SWIP-74 (BPS-lite) with the admin's service kinds of SWIP-87 +// (explicit multi-publisher): EOS, ROSTER and the Roster payload. Field and enum +// numbers are shared with SWIP-60, which extends these messages by adding fields +// and values only. +syntax = "proto3"; + +package bps; + +option go_package = "pb"; + +// What the topic binds to. One binding. +enum TopicBinding { + TOPIC_BINDING_UNSPECIFIED = 0; // invalid on the wire + FEED_TOPIC = 1; // the stream is a feed on the topic +} + +// The cohort's identity, keyed by its canonical serialisation. +message CohortSpec { + bytes topic = 1; // 32 bytes + TopicBinding binding = 2; // FEED_TOPIC + bytes principal = 3; // 20 bytes, required: the admin +} + +// Peer -> broker: the first frame on a fresh stream. +message Join { + CohortSpec cohort = 1; + bytes owner = 2; // 20 bytes or absent: the owner of the stream's frames, the wallet address + // signing its feed update SOCs; absent, the stream can only listen +} + +enum Status { + STATUS_UNSPECIFIED = 0; // invalid on the wire + OK = 1; + FULL = 2; // a capacity bound + REJECTED = 3; // a spec value outside SWIP-74, or a malformed Join +} + +// Broker -> peer, answering Join. A non-OK Ack ends the stream. +message Ack { + Status status = 1; + bytes challenge = 2; // iff OK: 32 random bytes for this stream +} + +// What a chunk on this wire is. +enum Kind { + KIND_UNSPECIFIED = 0; // invalid on the wire + DATA = 1; // an update of the stream's session feed + AUTH = 2; // the empty chunk at index 0 of the session's AUTH feed: authenticates + // a pending stream, or a spectator stream a delivered roster names; + // a heartbeat on a publisher stream; never delivered + EOS = 3; // the empty chunk at index 0 of the admin's EOS feed: the channel is ended + ROSTER = 4; // from the admin: the publisher set as of this index; payload = Roster +} + +// The payload of a ROSTER chunk. +message Roster { + repeated bytes publishers = 1; // 20-byte addresses: the complete set excluding the admin +} + +// Every frame after the handshake. soc is the index-th update of the session feed: +// a single-owner chunk whose id is derived from kind, challenge and index (see bps.ID). +message Broadcast { + bytes soc = 1; + Kind kind = 2; + bytes challenge = 3; // 32 bytes + uint64 index = 4; +} diff --git a/pkg/bps/pb/doc.go b/pkg/bps/pb/doc.go new file mode 100644 index 00000000000..57b18ce87e9 --- /dev/null +++ b/pkg/bps/pb/doc.go @@ -0,0 +1,8 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate sh -c "protoc -I . -I \"$(go list -f '{{ .Dir }}' -m github.com/gogo/protobuf)/protobuf\" --gogofaster_out=. bps.proto" + +// Package pb holds only Protocol Buffer definitions and generated code. +package pb diff --git a/pkg/node/node.go b/pkg/node/node.go index b9a93612e3e..41d9e9a79d3 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -31,6 +31,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/accounting" "github.com/ethersphere/bee/v2/pkg/addressbook" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/config" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/feeds/factory" @@ -114,6 +115,7 @@ type Bee struct { pullSyncCloser io.Closer pssCloser io.Closer gsocCloser io.Closer + bpsCloser io.Closer transactionMonitorCloser io.Closer transactionCloser io.Closer listenerCloser io.Closer @@ -167,6 +169,7 @@ type Options struct { AutoTLSDomain string AutoTLSRegistrationEndpoint string FullNodeMode bool + BpsBroker bool LightNodeLimit int GasLimitFallback uint64 Logger log.Logger @@ -303,6 +306,10 @@ func NewBee( } }(b) + if !o.FullNodeMode && o.BpsBroker { + return nil, fmt.Errorf("bps broker is only allowed for full nodes") + } + if !o.FullNodeMode && o.ReserveCapacityDoubling != 0 { return nil, fmt.Errorf("reserve capacity doubling is only allowed for full nodes") } @@ -1106,6 +1113,14 @@ func NewBee( b.pssCloser = pssService b.gsocCloser = gsocService + bpsService := bps.New(p2ps, p2ps, o.BpsBroker, logger, bps.DefaultOptions) + b.bpsCloser = bpsService + if o.BpsBroker { + if err = p2ps.AddProtocol(bpsService.Protocol()); err != nil { + return nil, fmt.Errorf("bps protocol: %w", err) + } + } + validStamp := postage.ValidStamp(batchStore) // metrics exposed on the status protocol @@ -1381,6 +1396,7 @@ func NewBee( Resolver: multiResolver, Pss: pssService, Gsoc: gsocService, + Bps: bpsService, FeedFactory: feedFactory, Post: post, AccessControl: accesscontrol, @@ -1395,6 +1411,7 @@ func NewBee( if o.APIAddr != "" { // register metrics from components apiService.MustRegisterMetrics(p2ps.Metrics()...) + apiService.MustRegisterMetrics(bpsService.Metrics()...) apiService.MustRegisterMetrics(pingPong.Metrics()...) apiService.MustRegisterMetrics(acc.Metrics()...) apiService.MustRegisterMetrics(localStore.Metrics()...) @@ -1487,6 +1504,7 @@ func (b *Bee) shutdownClosers() []namedCloser { return []namedCloser{ {b.pssCloser, "pss"}, {b.gsocCloser, "gsoc"}, + {b.bpsCloser, "bps"}, {b.pusherCloser, "pusher"}, {b.pullerCloser, "puller"}, {b.accountingCloser, "accounting"},