feat(pushsync): placement check that verifies where chunks land

config/light-testnet.yaml

@@ -0,0 +1,75 @@
+# bee-light-testnet: 20 full nodes on an isolated testnet (swarm1), used for A/B runs.
+# The bootnode has no ingress and is not part of the cluster definition.
+clusters:
+  bee-light-testnet:
+    _inherit: ""
+    name: bee
+    namespace: bee-light-testnet
+    disable-namespace: true
+    api-domain: lightnet.testnet.internal
+    api-domain-internal: bee-light-testnet.svc.swarm1.local:1633
+    api-insecure-tls: true
+    api-scheme: http
+    funding:
+      eth: 0.1
+      gbzz: 2.0
+      bzz: 5.0
+    node-groups:
+      bee-1:
+        mode: node
+        bee-config: light-testnet
+        config: ng-light-testnet
+        count: 5
+      bee-2:
+        mode: node
+        bee-config: light-testnet
+        config: ng-light-testnet
+        count: 5
+      bee-3:
+        mode: node
+        bee-config: light-testnet
+        config: ng-light-testnet
+        count: 5
+      bee-4:
+        mode: node
+        bee-config: light-testnet
+        config: ng-light-testnet
+        count: 5
+
+node-groups:
+  ng-light-testnet:
+    _inherit: ""
+
+bee-configs:
+  light-testnet:
+    _inherit: ""
+    full-node: true
+
+checks:
+  lt-pushsync-placement:
+    options:
+      chunks-per-node: 2 # per neighborhood, per uploader
+      mode: placement
+      postage-ttl: 24h
+      postage-depth: 22
+      postage-label: test-label
+      retry-delay: 5s
+      storage-radius: 2 # the radius the nodes report in /status
+      upload-nodes: [bee-1-2, bee-2-0, bee-3-0, bee-3-1, bee-4-3, bee-4-4]
+      upload-timeout: 60s
+    timeout: 60m
+    type: pushsync
+  lt-pushsync-placement-deferred:
+    options:
+      chunks-per-node: 2
+      deferred: true
+      mode: placement
+      postage-ttl: 24h
+      postage-depth: 22
+      postage-label: test-label
+      retry-delay: 30s # the pusher queue is busy with the background load
+      storage-radius: 2
+      upload-nodes: [bee-1-2, bee-2-0, bee-3-0, bee-3-1, bee-4-3, bee-4-4]
+      upload-timeout: 60s
+    timeout: 60m
+    type: pushsync

config/local.yaml

@@ -40,7 +40,7 @@ clusters:
       bee:
         bee-config: bee-local-dns
         config: local-dns
-        count: 3
+        count: 5
         mode: node
       light:
         bee-config: bee-local-light
@@ -207,10 +207,14 @@ bee-configs:
   bootnode-local:
     _inherit: "bee-local"
     bootnode-mode: true
+  # a forced storage radius splits the small cluster into neighborhoods, some
+  # of them empty, so that out-of-depth storing and shallow receipts can occur
   bee-local-dns:
     _inherit: "bee-local"
+    minimum-storage-radius: 3
   bootnode-local-dns:
     _inherit: "bee-local"
+    minimum-storage-radius: 3
     bootnode-mode: true
   bee-local-light:
     _inherit: "bee-local"
@@ -304,6 +308,33 @@ checks:
         - bootnode
     timeout: 5m
     type: pushsync
+  ci-pushsync-placement:
+    options:
+      chunks-per-node: 2 # per neighborhood, per uploader
+      mode: placement
+      postage-ttl: 24h
+      postage-depth: 21
+      postage-label: test-label
+      retry-delay: 3s
+      storage-radius: 3 # must match minimum-storage-radius of the full nodes
+      upload-node-count: 2
+      upload-timeout: 30s
+    timeout: 30m
+    type: pushsync
+  ci-pushsync-placement-deferred:
+    options:
+      chunks-per-node: 2 # per neighborhood, per uploader
+      deferred: true
+      mode: placement
+      postage-ttl: 24h
+      postage-depth: 21
+      postage-label: test-label
+      retry-delay: 3s
+      storage-radius: 3 # must match minimum-storage-radius of the full nodes
+      upload-node-count: 2
+      upload-timeout: 30s
+    timeout: 30m
+    type: pushsync
   ci-pushsync-light-chunks:
     options:
       chunks-per-node: 3

pkg/bee/api/chunks.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"io"
 	"net/http"
+	"strconv"
 
 	"github.com/ethersphere/bee/v2/pkg/swarm"
 )
@@ -36,6 +37,9 @@ func (c *ChunksService) Upload(ctx context.Context, data []byte, o UploadOptions
 	if o.Direct {
 		h.Add(deferredUploadHeader, "false")
 	}
+	if o.Tag != 0 {
+		h.Add(swarmTagHeader, strconv.FormatUint(o.Tag, 10))
+	}
 	h.Add(postageStampBatchHeader, o.BatchID)
 	err := c.client.requestWithHeader(ctx, http.MethodPost, "/"+apiVersion+"/chunks", h, bytes.NewReader(data), &resp)
 	return resp, err

pkg/bee/api/node.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 
@@ -95,6 +96,36 @@ func (n *NodeService) HasChunk(ctx context.Context, a swarm.Address) (bool, erro
 	return true, nil
 }
 
+// HasChunkLocal reports whether the chunk is in the node's local store. Unlike
+// HasChunk it uses HEAD, which never retrieves the chunk from the network and
+// so does not change what the node stores.
+func (n *NodeService) HasChunkLocal(ctx context.Context, a swarm.Address) (bool, error) {
+	err := n.client.request(ctx, http.MethodHead, "/chunks/"+a.String(), nil, nil)
+	if IsHTTPStatusErrorCode(err, http.StatusNotFound) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+// Metrics returns the node's metrics in the Prometheus text format.
+func (n *NodeService) Metrics(ctx context.Context) (string, error) {
+	r, err := n.client.requestData(ctx, http.MethodGet, "/metrics", nil, nil)
+	if err != nil {
+		return "", err
+	}
+	defer r.Close()
+
+	b, err := io.ReadAll(r)
+	if err != nil {
+		return "", err
+	}
+
+	return string(b), nil
+}
+
 // Health represents node's health
 type Health struct {
 	Status string `json:"status"`

pkg/bee/client.go

@@ -270,6 +270,17 @@ func (c *Client) HasChunk(ctx context.Context, a swarm.Address) (bool, error) {
 	return c.api.Node.HasChunk(ctx, a)
 }
 
+// HasChunkLocal reports whether the node has the chunk in its local store,
+// without retrieving it from the network.
+func (c *Client) HasChunkLocal(ctx context.Context, a swarm.Address) (bool, error) {
+	return c.api.Node.HasChunkLocal(ctx, a)
+}
+
+// Metrics returns the node's metrics in the Prometheus text format.
+func (c *Client) Metrics(ctx context.Context) (string, error) {
+	return c.api.Node.Metrics(ctx)
+}
+
 func (c *Client) HasChunks(ctx context.Context, a []swarm.Address) (has []bool, count int, err error) {
 	has = make([]bool, len(a))
 	for i, addr := range a {