diff --git a/docs/guides/observability.md b/docs/guides/observability.md index 8969b174fe..e4e0ca0358 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -302,6 +302,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `atelet.snapshot.upload.overdue.actors` | atelet | up/down counter | actors on the node with an undurable local snapshot past `--snapshot-upload-delay` waiting for an upload to finish, bucketed by time overdue (label `ate.snapshot.upload_overdue_band`: `under_1m`, `1m_to_5m`, `5m_to_15m`, `over_15m`) | | `atelet.snapshot.undurable.age.max` | atelet | gauge | age of the oldest local snapshot on the node that is not durable yet, counted from when the actor's state stopped being durable, or 0 when every local snapshot is durable; a healthy node stays below `--snapshot-upload-delay` plus one upload | | `ate.imagecache.requests` | atelet | counter | image lookups in the node-local image cache, by outcome (`ate.imagecache.outcome`), with `error.type` on the `error` outcome. A miss pays for the pull and the unpack, so the hit ratio per node is a leading indicator of resume latency | +| `ate.imagecache.size` | atelet | up/down counter | bytes the node-local image cache's layers hold, read on each collection (no labels); compare with the volume's usage to see how much of the disk the image cache holds | | `ate.filecache.size` | atelet | up/down counter | bytes of disk a node-local file cache occupies, by `ate.filecache.name`, read on each collection; compare with `--sandbox-asset-cache-max-bytes` | | `ate.filecache.evictions` | atelet | counter | entries the eviction passes removed from a node-local file cache, by `ate.filecache.name`; a steady rate at the budget means the cache thrashes and actors pay for downloads again | | `ate.filecache.gc.passes` | atelet | counter | eviction passes over a node-local file cache, by `ate.filecache.name`, that had work to do, by outcome (`ate.filecache.gc.outcome`: `complete`, `shortfall`, `skipped`); a pass within budget is not counted | @@ -334,6 +335,9 @@ For `ate.imagecache.requests`: * `ate.imagecache.outcome` is `hit` when the node holds a complete image record — every layer directory the record names is present — and `miss` when the lookup must pull. A failed lookup is neither: `error` is a failed lookup whatever the cause, and `cancelled` or `timeout` is the caller giving up, as on `ate.router.outcome`. So the hit ratio is `hit / (hit + miss)`, with failures and abandoned lookups out of the denominator. * `error.type` is present only on the `error` outcome, and carries the registry's own HTTP status for its rejection, from a fixed set: `401`, `403`, `404`, `429`, `500`, `502`, `503`, `504`. The set is an allow-list because the registry client reports whatever the remote returned. Each other status, and each failure that carries no status, reports `_OTHER`. +For `ate.imagecache.size`: +* It sums the size recorded for each layer when it was unpacked, so it counts file sizes, not allocated blocks. It is measured on each collection, whatever evicts the cache. + For the `ate.filecache.*` instruments, one series per cache under `ate.filecache.name`: `sandbox_asset` for the sandbox asset cache and `snapshot` for the snapshot cache (see the [Sandbox Asset Cache](../concepts/sandbox-asset-cache.md) page): * The cache evicts only what nothing links: an actor's staged assets and each worker's default runtime are hard links into the cache, and an entry with a link stays whatever the pressure. `ate.filecache.gc.outcome` is `shortfall` when a pass could not reach the budget for that reason; the size then measures what the actors and the defaults on the node need, not waste. `skipped` means atelet could not read the cache directory and did nothing. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 93d4214b07..58b85bd2c2 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -1376,6 +1376,27 @@ groups: The permitted values are 401, 403, 404, 429, 500, 502, 503 and 504. Each other status, and each failure with no status, reports _OTHER. + - id: metric.ate.imagecache.size + type: metric + metric_name: ate.imagecache.size + instrument: updowncounter + unit: By + stability: development + brief: The bytes that the layers of the image cache on the node hold. + note: > + atelet sums the size recorded for each layer on every collection. The + sizes are file sizes, not allocated blocks. This instrument has no + labels. Compare the value with the volume's usage to see how much of the + disk the image cache holds, and with --image-cache-max-bytes when that + is set. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [saturation] + code_anchor: internal/imagecache/metrics.go + cuj: The disk of a worker node fills. Is it the image cache? + attributes: [] + - id: metric.ate.filecache.size type: metric metric_name: ate.filecache.size diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index 03cf473c4e..c23ff83737 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -223,9 +223,10 @@ blind_spots: - area: image cache cost and eviction code: internal/imagecache brief: > - The hit and miss counter exists. But the garbage collector (the 85% and - 80% limits) and the time to prepare an image send nothing. Thus you cannot - see the disk pressure from the layer pool. + The hit and miss counter and the size of the layer pool exist. But the + garbage collector (the 85% and 80% limits) and the time to prepare an + image send nothing. Thus you can see how much of the disk the layer pool + holds, but not what the collector evicts. - area: podcertcontroller code: cmd/podcertcontroller brief: This component has no metrics. diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index d49112c6c9..7ce464cfa8 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -284,7 +284,8 @@ func WithPullTimeout(d time.Duration) Option { return func(s *Store) { s.pullTimeout = d } } -// WithMeter attaches the meter the store reports ate.imagecache.requests on. +// WithMeter attaches the meter the store reports ate.imagecache.requests and +// ate.imagecache.size on. // Without it the store records nothing, so a caller with no metrics pipeline // needs no meter provider. func WithMeter(m metric.Meter) Option { @@ -327,6 +328,9 @@ func New(root string, opts ...Option) (*Store, error) { return nil, err } s.requests = requests + if err := s.observeSize(s.meter); err != nil { + return nil, err + } } for _, d := range []string{s.layersDir(), s.manifestsDir()} { diff --git a/internal/imagecache/metrics.go b/internal/imagecache/metrics.go index 968984a244..523e2685e4 100644 --- a/internal/imagecache/metrics.go +++ b/internal/imagecache/metrics.go @@ -27,7 +27,10 @@ import ( "go.ate.dev/substrate/internal/ateattr" ) -const requestsMetric = "ate.imagecache.requests" +const ( + requestsMetric = "ate.imagecache.requests" + sizeMetric = "ate.imagecache.size" +) // errTypeOther is the OTel registry's fallback, used for a failure that // carries no status of its own. It is spelled _OTHER, not the "unknown" of the @@ -57,6 +60,29 @@ func newRequestsCounter(meter metric.Meter) (metric.Int64Counter, error) { return counter, nil } +// observeSize reports the layer pool's size as ate.imagecache.size on every +// collection. Measuring reads one size file per layer, so it doesn't depend on +// any eviction loop having run. +func (s *Store) observeSize(meter metric.Meter) error { + _, err := meter.Int64ObservableUpDownCounter( + sizeMetric, + metric.WithUnit("By"), + metric.WithDescription("Bytes the layers of the node-local image cache hold."), + metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { + n, err := s.CacheSize() + if err != nil { + return err + } + o.Observe(n) + return nil + }), + ) + if err != nil { + return fmt.Errorf("create %s up/down counter: %w", sizeMetric, err) + } + return nil +} + // recordRequest counts one EnsureImage lookup. A store with no meter records // nothing. A failure replaces the hit-or-miss outcome the caller passed, and // only the Error outcome carries an error.type. diff --git a/internal/imagecache/metrics_test.go b/internal/imagecache/metrics_test.go index 756b4dacf2..10bb2985c7 100644 --- a/internal/imagecache/metrics_test.go +++ b/internal/imagecache/metrics_test.go @@ -245,3 +245,69 @@ func TestRequestsWithoutMeterIsNoOp(t *testing.T) { t.Fatalf("EnsureImage: %v", err) } } + +// collectSize returns the ate.imagecache.size value, checking its shape. +func collectSize(t *testing.T, reader *sdkmetric.ManualReader) int64 { + t.Helper() + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("collect: %v", err) + } + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != sizeMetric { + continue + } + if m.Unit != "By" { + t.Errorf("unit = %q, want By", m.Unit) + } + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + t.Fatalf("data type = %T, want Sum[int64]", m.Data) + } + if sum.IsMonotonic { + t.Error("IsMonotonic = true, want false (an updowncounter)") + } + if len(sum.DataPoints) != 1 || sum.DataPoints[0].Attributes.Len() != 0 { + t.Fatalf("data points = %+v, want one with no labels", sum.DataPoints) + } + return sum.DataPoints[0].Value + } + } + t.Fatalf("%s not collected", sizeMetric) + return 0 +} + +// TestSizeIsMeasuredOnCollection checks the size follows pulls and evictions +// on the next collection, with no eviction loop involved. +func TestSizeIsMeasuredOnCollection(t *testing.T) { + _, host := newTestRegistry(t) + ref := host + "/test/size:latest" + pushOneLayerImage(t, ref) + + store, reader := newMeteredStore(t, WithMinAge(0)) + ctx := context.Background() + + if got := collectSize(t, reader); got != 0 { + t.Errorf("empty cache size = %d, want 0", got) + } + if _, err := store.EnsureImage(ctx, ref); err != nil { + t.Fatalf("EnsureImage: %v", err) + } + want, err := store.CacheSize() + if err != nil { + t.Fatalf("CacheSize: %v", err) + } + if want == 0 { + t.Fatal("CacheSize = 0 after a pull, want the layer's size") + } + if got := collectSize(t, reader); got != want { + t.Errorf("size after pull = %d, want %d", got, want) + } + if _, err := store.EvictUnused(ctx, want, false); err != nil { + t.Fatalf("EvictUnused: %v", err) + } + if got := collectSize(t, reader); got != 0 { + t.Errorf("size after evicting everything = %d, want 0", got) + } +}