diff --git a/Cargo.lock b/Cargo.lock index ee815167cb4..d688417cbbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5349,6 +5349,7 @@ dependencies = [ "moq-net", "serde", "serde_json", + "serde_with", "thiserror 2.0.21", "tokio", "tracing", diff --git a/demo/relay/localhost.toml b/demo/relay/localhost.toml index b68064835af..6296b741266 100644 --- a/demo/relay/localhost.toml +++ b/demo/relay/localhost.toml @@ -30,7 +30,7 @@ public = "**" # # .stats/node/local/host # -# carrying totals.json, publisher.json, subscriber.json, and sessions.json tracks. See +# carrying totals.json and group.json tracks. See # doc/bin/relay/config.md for the field semantics. [stats] enabled = true diff --git a/demo/web/src/stats.ts b/demo/web/src/stats.ts index c912cc70c54..814a3446b92 100644 --- a/demo/web/src/stats.ts +++ b/demo/web/src/stats.ts @@ -8,23 +8,24 @@ * let you drill into one. A restarted relay announces a new epoch at the same * path, which arrives as an end then a start, so its history starts over. * - * Per-node tracks we read: - * publisher.json egress (relay -> downstream viewers) - * subscriber.json ingress (upstream publishers -> relay) - * sessions.json sessions by stats label + * Per node we read `group.json`, the whole node's group at the relay's default + * `depth = 0`: `{ epoch, tiers: { : { paths, labels } } }`, a JSON snapshot + * with merge-patch deltas. From the default tier we take each path's `publisher` + * (egress, relay -> downstream viewers) and `subscriber` (ingress, upstream + * publishers -> relay), and each stats label's sessions. * - * Each frame is `{ "": Snapshot }`. Counters are cumulative; - * "active" = started - ended. The relay includes every entry it still holds - * counters for, idle ones too, so the latest frame is a snapshot of now. We sample the aggregate on an interval - * to derive per-second throughput rates for the charts. A relay built after the - * started/ended rename still writes the legacy names beside the new ones; this - * dashboard prefers the canonical spelling and falls back so it also reads an - * older relay. + * Counters are cumulative; "active" = started - ended. The relay includes every + * entry it still holds counters for, idle ones too, so the latest frame is a + * snapshot of now. We sample the aggregate on an interval to derive per-second + * throughput rates for the charts. A relay built after the started/ended rename + * still writes the legacy names beside the new ones; this dashboard prefers the + * canonical spelling and falls back so it also reads an older relay. */ import "./highlight"; import { Milli, Nano, wall } from "@moq/clock"; import { Net, Signals } from "@moq/hang"; +import * as Json from "@moq/json"; const RELAY_URL = import.meta.env.VITE_RELAY_URL ?? "http://localhost:4443"; @@ -78,9 +79,32 @@ interface SessionCounters { type SessionFrame = Record; interface NodeStats { - egress: BroadcastFrame; // publisher.json - ingress: BroadcastFrame; // subscriber.json - sessions: SessionFrame; // sessions.json + egress: BroadcastFrame; // each path's `publisher` + ingress: BroadcastFrame; // each path's `subscriber` + sessions: SessionFrame; // `labels` +} + +// One tier of a `group.json` frame. +interface GroupTier { + paths?: Record; + labels?: SessionFrame; +} + +interface GroupFrame { + tiers?: Record; +} + +// The default tier's egress, ingress, and sessions from a group frame. +function nodeStatsOf(frame: GroupFrame): NodeStats { + const tier = frame.tiers?.[""] ?? {}; + const egress: BroadcastFrame = {}; + const ingress: BroadcastFrame = {}; + for (const [path, usage] of Object.entries(tier.paths ?? {})) { + // The relay leaves out a side that never moved. + if (usage.publisher) egress[path] = usage.publisher; + if (usage.subscriber) ingress[path] = usage.subscriber; + } + return { egress, ingress, sessions: tier.labels ?? {} }; } // Canonical spelling wins when a frame carries both names. @@ -176,24 +200,18 @@ function subscribeNode(effect: Signals.Effect, origin: Net.Origin.Table, path: N const consumer = request.active.peek(); if (!consumer) return; - const sub = (trackName: string, key: K) => { - const track = consumer.track(trackName).subscribe().ordered(); - effect.cleanup(() => track.close()); - effect.spawn(async () => { - for (;;) { - const data = await effect.race(track.readJson()); - if (data === undefined) break; - nodeStats.mutate((s) => { - const cur = s[node]; - if (cur) cur[key] = (data ?? {}) as NodeStats[K]; - }); - } - }); - }; - - sub("publisher.json", "egress"); - sub("subscriber.json", "ingress"); - sub("sessions.json", "sessions"); + const track = consumer.track("group.json").subscribe(); + effect.cleanup(() => track.close()); + const group = new Json.Snapshot.Consumer({ track }); + effect.spawn(async () => { + for (;;) { + const state = await effect.race(group.latest()); + if (state === undefined) break; + nodeStats.mutate((s) => { + if (s[node]) s[node] = nodeStatsOf(state.value ?? {}); + }); + } + }); } // ---- Aggregation ---------------------------------------------------------- diff --git a/doc/bin/inspect.md b/doc/bin/inspect.md index 94f55ce0f4b..db8ecf3a57a 100644 --- a/doc/bin/inspect.md +++ b/doc/bin/inspect.md @@ -69,15 +69,16 @@ Dial the `.stats` path, which the token must cover: ```bash moq --connect http://localhost:4443/.stats announced -moq --connect http://localhost:4443/.stats --broadcast node/local/host fetch publisher.json | jq -curl http://localhost:4443/fetch/.stats/node/local/host/publisher.json | jq +moq --connect http://localhost:4443/.stats --broadcast node/local/host fetch totals.json | jq +curl http://localhost:4443/fetch/.stats/node/local/host/totals.json | jq ``` The node name (`local/host` here) is `stats.node`. Each fetch returns the newest snapshot of cumulative counters. `totals.json` sums each tier's traffic -and sessions, `publisher.json` is egress per broadcast path, -`subscriber.json` is ingress, and `sessions.json` counts sessions per stats -label. A subscriber may also ask for `/prefix.json`, one broadcast -prefix rolled up with each child, produced only while held. Fetch twice and divide by the interval for a -rate. The `.json.z` twins carry compressed patches, which these tools print as -raw bytes. [Stats](/concept/stats) describes every field. +and sessions. `group.json` (`/group.json` at `stats.depth` 1 or more) +carries a group's traffic per broadcast path and sessions per stats label: its +newest group is a full snapshot followed by merge-patch deltas, one per line. +A subscriber may also ask for `/prefix.json`, one broadcast prefix rolled +up with each child, produced only while held. Fetch twice and divide by the +interval for a rate. The `.json.z` twins carry compressed patches, which these +tools print as raw bytes. [Stats](/concept/stats) describes every field. diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 7bde4e3c424..7659f433533 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -350,20 +350,22 @@ group of every track is always kept, even past `capacity`. ```toml [stats] enabled = true -prefix = ".stats" # Broadcasts appear under /node/. +prefix = ".stats" # The broadcast appears at /node/. interval = "1s" # Time between snapshots. Default. node = "sjc/1" # Disambiguates relays sharing a cluster. -depth = 1 # Also bucket by the first N path segments (per tenant). -linger = "5m" # Keep an empty group's broadcast announced this long. Default. +depth = 1 # One group track per first N path segments (per tenant). +linger = "5m" # Keep an empty group's track listed this long. Default. ``` Each node publishes its traffic and session counters as MoQ tracks, split by a **tier** label from the auth server's grant (`--cluster-tier` for links this relay dials and LAN peers it admits), which is what makes billing per customer or per region possible. -Each run, and each group returning after its linger, announces under a fresh -[epoch](/concept/moq-lite#publisher-epochs), so a restart is a new broadcast at -the same path. Each group's totals count everything since its epoch began, so -billing reads those; per-path traffic and per-label sessions and HTTP requests are for +Each run announces under a fresh [epoch](/concept/moq-lite#publisher-epochs), +so a restart is a new broadcast at the same path. Each group is a +[track](/concept/stats#group-tracks) of that broadcast, and one returning after +its linger restarts its counters under a new epoch without an announce. The +node's totals count everything since its epoch began, so billing reads those; +a group's per-path traffic and per-label sessions and HTTP requests are for dashboards. Session counters report how close sessions come to the per-session limits, past which one is closed with `TOO_MANY_REQUESTS`. [Stats](/concept/stats) describes the paths, tracks, and encodings; read them diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 4c3c208fc0d..7dd336193af 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -145,99 +145,108 @@ and reads `unknown`; only its viewers' playout counters can show the damage. ## Broadcasts -Each node publishes under a prefix, `.stats` by default, which +Each node publishes one broadcast under a prefix, `.stats` by default, which [moq-lite](/concept/moq-lite) hides from announce listings that don't ask for it. Traffic under the prefix is never counted, so serving stats doesn't generate more stats. ```text -/node/ depth 0: one broadcast per node -//node/ depth N: one broadcast per group per node +/node/ ``` - `` tells relays sharing a cluster apart. It may span several segments (`sjc/1`), and is omitted along with its slash when unset: `/node`. -- `` is the first `depth` segments of each broadcast path (for traffic) - or stats label (for sessions), so a consumer can scope an announce to one - tenant. A path shorter than `depth` groups under all of its segments. - The literal `node` segment leaves room for sibling categories under the same - prefix, so a consumer skips any path without `node` where it expects one. A - group segment literally named `node` is ambiguous; don't use one. - -Each announcement carries a fresh [publisher epoch](/concept/moq-lite#publisher-epochs) -on its route, so a restarted node or a group returning from idle is a new -broadcast at the same path: it replaces the old one, and no relay serves it -groups cached under the old epoch. A reader treats each epoch as its own set of -counters. The epoch rides moq-lite 07 announcements, and moq-transport draft 17 -and later with the [epoch extension](/draft/moq-epoch); over anything else a -reader sees the same change as an end and a start at the path. + prefix, so a consumer skips any path without `node` where it expects one. + +The broadcast stays announced for the producer's life. Each announcement +carries a fresh [publisher epoch](/concept/moq-lite#publisher-epochs) on its +route, so a restarted node is a new broadcast at the same path: it replaces the +old one, and no relay serves it groups cached under the old epoch. A reader +treats each epoch as its own set of counters. The epoch rides moq-lite 07 +announcements, and moq-transport draft 17 and later with the +[epoch extension](/draft/moq-epoch); over anything else a reader sees the same +change as an end and a start at the path. An aggregating reader pins each node's subscription to its epoch, and when one publisher is reached through both kinds of link (with and without an epoch), it carries only that path's outgoing counters across the change. -At depth 0 the broadcast stays announced for the producer's life. At depth -1 or more, a group's broadcast is announced while that group has entries, and -for a linger (five minutes by default) after its last one leaves. A group that -returns within the linger keeps its broadcast and epoch, so viewer churn -doesn't unannounce and re-announce it across the mesh, and its totals -continue. Once the linger elapses with the group still empty, the broadcast is -unannounced and its counters dropped; a group that returns later announces a -new epoch counted from zero. Holding a stats track doesn't count as an entry: -every track ends with the broadcast, including presence, prefix, and requested -tier tracks a reader still holds, and the reader follows the group's next -announcement. -Within one epoch, group numbers keep increasing across recreated tracks and -may have gaps. +Every group shares the node's broadcast, so a token scoped to the stats prefix +reads every group's tracks; scope by group with an aggregating reader instead. ## Tracks Traffic is split by **tier**, an arbitrary label (a billing class, a region) the relay takes from the auth grant, or from `--cluster-tier` for links it -dials and LAN peers it admits. The default tier's label is empty. Each track -comes in two encodings: +dials and LAN peers it admits. The default tier's label is empty. Every track +but a group's comes in two encodings: | Track | Frame keyed by | Entry | | --- | --- | --- | -| `totals.json` | tier | [Totals](#totals) for the whole group | -| `[/]publisher.json` | broadcast path | [Traffic](#traffic) this node sent (egress) | -| `[/]subscriber.json` | broadcast path | [Traffic](#traffic) this node received (ingress) | -| `[/]sessions.json` | stats label | [Presence](#presence): sessions connected now | +| `totals.json` | tier | [Totals](#totals) for the whole node | +| `[/]group.json` | | [Group](#group): one group's totals, paths, and labels, every tier | | `[