From ac6c8dca6c81d8329d9e2800dc9da72484a55596 Mon Sep 17 00:00:00 2001 From: Ivy Date: Mon, 16 Nov 2026 17:03:03 -0800 Subject: [PATCH] imagecache: report the image cache size Nothing shows how much of a node's disk the image cache holds. Add ate.imagecache.size, the sum of the recorded layer sizes, measured on each collection. It lives in the image cache store rather than the GC loop, so it keeps reporting whatever evicts the cache. Signed-off-by: Ivy --- docs/guides/observability.md | 4 ++ docs/metrics/registry/metrics.yaml | 21 +++++++++ docs/metrics/substrate.yaml | 7 +-- internal/imagecache/imagecache.go | 6 ++- internal/imagecache/metrics.go | 28 +++++++++++- internal/imagecache/metrics_test.go | 66 +++++++++++++++++++++++++++++ 6 files changed, 127 insertions(+), 5 deletions(-) diff --git a/docs/guides/observability.md b/docs/guides/observability.md index 8969b174fe..e4e0ca0358 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -302,6 +302,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `atelet.snapshot.upload.overdue.actors` | atelet | up/down counter | actors on the node with an undurable local snapshot past `--snapshot-upload-delay` waiting for an upload to finish, bucketed by time overdue (label `ate.snapshot.upload_overdue_band`: `under_1m`, `1m_to_5m`, `5m_to_15m`, `over_15m`) | | `atelet.snapshot.undurable.age.max` | atelet | gauge | age of the oldest local snapshot on the node that is not durable yet, counted from when the actor's state stopped being durable, or 0 when every local snapshot is durable; a healthy node stays below `--snapshot-upload-delay` plus one upload | | `ate.imagecache.requests` | atelet | counter | image lookups in the node-local image cache, by outcome (`ate.imagecache.outcome`), with `error.type` on the `error` outcome. A miss pays for the pull and the unpack, so the hit ratio per node is a leading indicator of resume latency | +| `ate.imagecache.size` | atelet | up/down counter | bytes the node-local image cache's layers hold, read on each collection (no labels); compare with the volume's usage to see how much of the disk the image cache holds | | `ate.filecache.size` | atelet | up/down counter | bytes of disk a node-local file cache occupies, by `ate.filecache.name`, read on each collection; compare with `--sandbox-asset-cache-max-bytes` | | `ate.filecache.evictions` | atelet | counter | entries the eviction passes removed from a node-local file cache, by `ate.filecache.name`; a steady rate at the budget means the cache thrashes and actors pay for downloads again | | `ate.filecache.gc.passes` | atelet | counter | eviction passes over a node-local file cache, by `ate.filecache.name`, that had work to do, by outcome (`ate.filecache.gc.outcome`: `complete`, `shortfall`, `skipped`); a pass within budget is not counted | @@ -334,6 +335,9 @@ For `ate.imagecache.requests`: * `ate.imagecache.outcome` is `hit` when the node holds a complete image record — every layer directory the record names is present — and `miss` when the lookup must pull. A failed lookup is neither: `error` is a failed lookup whatever the cause, and `cancelled` or `timeout` is the caller giving up, as on `ate.router.outcome`. So the hit ratio is `hit / (hit + miss)`, with failures and abandoned lookups out of the denominator. * `error.type` is present only on the `error` outcome, and carries the registry's own HTTP status for its rejection, from a fixed set: `401`, `403`, `404`, `429`, `500`, `502`, `503`, `504`. The set is an allow-list because the registry client reports whatever the remote returned. Each other status, and each failure that carries no status, reports `_OTHER`. +For `ate.imagecache.size`: +* It sums the size recorded for each layer when it was unpacked, so it counts file sizes, not allocated blocks. It is measured on each collection, whatever evicts the cache. + For the `ate.filecache.*` instruments, one series per cache under `ate.filecache.name`: `sandbox_asset` for the sandbox asset cache and `snapshot` for the snapshot cache (see the [Sandbox Asset Cache](../concepts/sandbox-asset-cache.md) page): * The cache evicts only what nothing links: an actor's staged assets and each worker's default runtime are hard links into the cache, and an entry with a link stays whatever the pressure. `ate.filecache.gc.outcome` is `shortfall` when a pass could not reach the budget for that reason; the size then measures what the actors and the defaults on the node need, not waste. `skipped` means atelet could not read the cache directory and did nothing. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 93d4214b07..58b85bd2c2 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -1376,6 +1376,27 @@ groups: The permitted values are 401, 403, 404, 429, 500, 502, 503 and 504. Each other status, and each failure with no status, reports _OTHER. + - id: metric.ate.imagecache.size + type: metric + metric_name: ate.imagecache.size + instrument: updowncounter + unit: By + stability: development + brief: The bytes that the layers of the image cache on the node hold. + note: > + atelet sums the size recorded for each layer on every collection. The + sizes are file sizes, not allocated blocks. This instrument has no + labels. Compare the value with the volume's usage to see how much of the + disk the image cache holds, and with --image-cache-max-bytes when that + is set. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [saturation] + code_anchor: internal/imagecache/metrics.go + cuj: The disk of a worker node fills. Is it the image cache? + attributes: [] + - id: metric.ate.filecache.size type: metric metric_name: ate.filecache.size diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index 03cf473c4e..c23ff83737 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -223,9 +223,10 @@ blind_spots: - area: image cache cost and eviction code: internal/imagecache brief: > - The hit and miss counter exists. But the garbage collector (the 85% and - 80% limits) and the time to prepare an image send nothing. Thus you cannot - see the disk pressure from the layer pool. + The hit and miss counter and the size of the layer pool exist. But the + garbage collector (the 85% and 80% limits) and the time to prepare an + image send nothing. Thus you can see how much of the disk the layer pool + holds, but not what the collector evicts. - area: podcertcontroller code: cmd/podcertcontroller brief: This component has no metrics. diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index d49112c6c9..7ce464cfa8 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -284,7 +284,8 @@ func WithPullTimeout(d time.Duration) Option { return func(s *Store) { s.pullTimeout = d } } -// WithMeter attaches the meter the store reports ate.imagecache.requests on. +// WithMeter attaches the meter the store reports ate.imagecache.requests and +// ate.imagecache.size on. // Without it the store records nothing, so a caller with no metrics pipeline // needs no meter provider. func WithMeter(m metric.Meter) Option { @@ -327,6 +328,9 @@ func New(root string, opts ...Option) (*Store, error) { return nil, err } s.requests = requests + if err := s.observeSize(s.meter); err != nil { + return nil, err + } } for _, d := range []string{s.layersDir(), s.manifestsDir()} { diff --git a/internal/imagecache/metrics.go b/internal/imagecache/metrics.go index 968984a244..523e2685e4 100644 --- a/internal/imagecache/metrics.go +++ b/internal/imagecache/metrics.go @@ -27,7 +27,10 @@ import ( "go.ate.dev/substrate/internal/ateattr" ) -const requestsMetric = "ate.imagecache.requests" +const ( + requestsMetric = "ate.imagecache.requests" + sizeMetric = "ate.imagecache.size" +) // errTypeOther is the OTel registry's fallback, used for a failure that // carries no status of its own. It is spelled _OTHER, not the "unknown" of the @@ -57,6 +60,29 @@ func newRequestsCounter(meter metric.Meter) (metric.Int64Counter, error) { return counter, nil } +// observeSize reports the layer pool's size as ate.imagecache.size on every +// collection. Measuring reads one size file per layer, so it doesn't depend on +// any eviction loop having run. +func (s *Store) observeSize(meter metric.Meter) error { + _, err := meter.Int64ObservableUpDownCounter( + sizeMetric, + metric.WithUnit("By"), + metric.WithDescription("Bytes the layers of the node-local image cache hold."), + metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { + n, err := s.CacheSize() + if err != nil { + return err + } + o.Observe(n) + return nil + }), + ) + if err != nil { + return fmt.Errorf("create %s up/down counter: %w", sizeMetric, err) + } + return nil +} + // recordRequest counts one EnsureImage lookup. A store with no meter records // nothing. A failure replaces the hit-or-miss outcome the caller passed, and // only the Error outcome carries an error.type. diff --git a/internal/imagecache/metrics_test.go b/internal/imagecache/metrics_test.go index 756b4dacf2..10bb2985c7 100644 --- a/internal/imagecache/metrics_test.go +++ b/internal/imagecache/metrics_test.go @@ -245,3 +245,69 @@ func TestRequestsWithoutMeterIsNoOp(t *testing.T) { t.Fatalf("EnsureImage: %v", err) } } + +// collectSize returns the ate.imagecache.size value, checking its shape. +func collectSize(t *testing.T, reader *sdkmetric.ManualReader) int64 { + t.Helper() + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("collect: %v", err) + } + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != sizeMetric { + continue + } + if m.Unit != "By" { + t.Errorf("unit = %q, want By", m.Unit) + } + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + t.Fatalf("data type = %T, want Sum[int64]", m.Data) + } + if sum.IsMonotonic { + t.Error("IsMonotonic = true, want false (an updowncounter)") + } + if len(sum.DataPoints) != 1 || sum.DataPoints[0].Attributes.Len() != 0 { + t.Fatalf("data points = %+v, want one with no labels", sum.DataPoints) + } + return sum.DataPoints[0].Value + } + } + t.Fatalf("%s not collected", sizeMetric) + return 0 +} + +// TestSizeIsMeasuredOnCollection checks the size follows pulls and evictions +// on the next collection, with no eviction loop involved. +func TestSizeIsMeasuredOnCollection(t *testing.T) { + _, host := newTestRegistry(t) + ref := host + "/test/size:latest" + pushOneLayerImage(t, ref) + + store, reader := newMeteredStore(t, WithMinAge(0)) + ctx := context.Background() + + if got := collectSize(t, reader); got != 0 { + t.Errorf("empty cache size = %d, want 0", got) + } + if _, err := store.EnsureImage(ctx, ref); err != nil { + t.Fatalf("EnsureImage: %v", err) + } + want, err := store.CacheSize() + if err != nil { + t.Fatalf("CacheSize: %v", err) + } + if want == 0 { + t.Fatal("CacheSize = 0 after a pull, want the layer's size") + } + if got := collectSize(t, reader); got != want { + t.Errorf("size after pull = %d, want %d", got, want) + } + if _, err := store.EvictUnused(ctx, want, false); err != nil { + t.Fatalf("EvictUnused: %v", err) + } + if got := collectSize(t, reader); got != 0 { + t.Errorf("size after evicting everything = %d, want 0", got) + } +}