From cd0b4d7286536ef63739289c2026a37a35acab09 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 28 Oct 2026 09:54:22 +0000 Subject: [PATCH 1/4] The Kubernetes credential provider exports metrics to the cluster collector The Deployment had no envFrom for ate-otel-config, so the OTLP exporter dialed localhost:4317. It now also sets the pod resource attributes like the other components. Signed-off-by: Eric Curtin --- .../internal/steps/credprovider_test.go | 21 +++++++++++++++++++ manifests/ate-install/ate-otel-config.yaml | 2 +- .../k8s-credential-provider.yaml | 21 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/cmd/ate-setup/internal/steps/credprovider_test.go b/cmd/ate-setup/internal/steps/credprovider_test.go index 1539f8ac3f..5a8b9f8aa7 100644 --- a/cmd/ate-setup/internal/steps/credprovider_test.go +++ b/cmd/ate-setup/internal/steps/credprovider_test.go @@ -15,6 +15,7 @@ package steps import ( + "slices" "testing" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -46,6 +47,9 @@ func TestK8sCredentialProviderManifestsAgree(t *testing.T) { if got := mountedConfigMaps(dep); len(got) != 1 || got[0] != k8sCredentialProviderPolicyConfigMap { t.Errorf("deployment/%s mounts ConfigMaps %v, want only %q", k8sCredentialProviderDeployment, got, k8sCredentialProviderPolicyConfigMap) } + if !slices.Contains(envFromConfigMaps(dep), otelConfigMap) { + t.Errorf("deployment/%s does not load configmap/%s through envFrom, so its metrics go to localhost:4317", k8sCredentialProviderDeployment, otelConfigMap) + } svc := findObject(provider, "Service", k8sCredentialProviderDeployment) if svc == nil { t.Fatalf("provider manifest has no service/%s", k8sCredentialProviderDeployment) @@ -202,3 +206,20 @@ func mountedConfigMaps(dep *unstructured.Unstructured) []string { } return names } + +// envFromConfigMaps returns the names of the ConfigMaps a Deployment's +// containers load through envFrom. +func envFromConfigMaps(dep *unstructured.Unstructured) []string { + containers, _, _ := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") + var names []string + for _, c := range containers { + sources, _, _ := unstructured.NestedSlice(c.(map[string]any), "envFrom") + for _, src := range sources { + name, found, _ := unstructured.NestedString(src.(map[string]any), "configMapRef", "name") + if found { + names = append(names, name) + } + } + } + return names +} diff --git a/manifests/ate-install/ate-otel-config.yaml b/manifests/ate-install/ate-otel-config.yaml index 0ee44e5053..17f5491061 100644 --- a/manifests/ate-install/ate-otel-config.yaml +++ b/manifests/ate-install/ate-otel-config.yaml @@ -14,7 +14,7 @@ # Single source of truth for the OTLP exporter settings shared by every control # plane component (ate-api-server, ate-controller, atelet, atenet-router, -# atenet-egress), which consume it via `envFrom`. ate-controller additionally propagates these values +# atenet-egress, k8s-credential-provider), which consume it via `envFrom`. ate-controller additionally propagates these values # to the worker pods it creates, which have no other way to receive them, # and atenet-router's --otlp-collector-address defaults to the endpoint below, # so Envoy's own spans follow it too. The exporter selection is the exception: diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml index 29f19338f0..03dc461f7a 100644 --- a/manifests/egress-credential-injection/k8s-credential-provider.yaml +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -89,6 +89,27 @@ spec: # Enforce the atespace→namespace authorization policy (default-deny). - "--namespace-policy-file=/etc/k8s-credential-provider/namespace-policy.yaml" - "--log-level=info" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) + # OTLP exporter settings shared by every control plane component; see + # manifests/ate-install/ate-otel-config.yaml. Without this the SDK + # falls back to localhost:4317 and every metric is lost. + envFrom: + - configMapRef: + name: ate-otel-config ports: - name: grpc containerPort: 50051 From 43837eddd91ab04fc819e4e38e88d517167095e5 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Thu, 5 Nov 2026 22:25:58 +0000 Subject: [PATCH 2/4] Restart the credential provider on --otlp-endpoint, fix docs Signed-off-by: Eric Curtin --- benchmarking/telemetry/README.md | 2 +- cmd/ate-setup/internal/steps/overlay.go | 2 +- cmd/ate-setup/internal/steps/overlay_test.go | 11 ++++++++++- docs/observability.md | 4 ++-- manifests/ate-install/ate-otel-config.yaml | 5 ++++- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/benchmarking/telemetry/README.md b/benchmarking/telemetry/README.md index a5d991b037..b254129ddc 100644 --- a/benchmarking/telemetry/README.md +++ b/benchmarking/telemetry/README.md @@ -46,7 +46,7 @@ go run ./cmd/ate-setup deploy ate-system \ `--otlp-endpoint` patches the `ate-otel-config` ConfigMap and restarts the workloads that read it. One patch is sufficient for `ateapi`, `ate-controller`, -`atelet`, and `atenet-router`, because each one reads the ConfigMap through +`atelet`, `atenet-router`, and `k8s-credential-provider`, because each one reads the ConfigMap through `envFrom`. `ate-controller` also copies the values to the worker pods that it creates. diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 401b5b2b0d..7fa99c1a29 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -603,7 +603,7 @@ const otelEndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT" // otelOverrideDeployments are the control plane Deployments that read // ate-otel-config. ate-controller additionally copies the values onto the // worker pods it creates, so one patch reaches the whole system. -var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router"} +var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "k8s-credential-provider"} // applyOtelEndpointOverride points all control plane telemetry at a different // collector for the duration of a measurement. See diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 7de3d99195..84944b3a7c 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -1402,9 +1402,15 @@ func TestApplyOtelEndpointOverride(t *testing.T) { Name: "atelet", }, } + provider := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: NamespaceAteSystem, + Name: "k8s-credential-provider", + }, + } e := &Env{ Cfg: &config.Config{OtlpEndpoint: endpoint}, - Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), atelet), + Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), atelet, provider), } if err := e.applyOtelEndpointOverride(t.Context()); err != nil { @@ -1420,6 +1426,9 @@ func TestApplyOtelEndpointOverride(t *testing.T) { if !restartedAt(t, e, "deployment", "ate-api-server") { t.Error("ate-api-server was not restarted") } + if !restartedAt(t, e, "deployment", "k8s-credential-provider") { + t.Error("k8s-credential-provider was not restarted") + } if !restartedAt(t, e, "daemonset", "atelet") { t.Error("the atelet DaemonSet was not restarted") } diff --git a/docs/observability.md b/docs/observability.md index ab93ba11b4..012aa254b9 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -424,7 +424,7 @@ To explore metrics locally: > **Note:** Storage is ephemeral (`emptyDir`), so metrics are lost when the Prometheus pod restarts. -> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics appear in neither: its pod carries no scrape annotation and does not read `ate-otel-config`, so its OTLP push goes to the SDK default `localhost:4317` and fails. +> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics appear in neither: its pod carries no scrape annotation, so it is not in this UI. Its OTLP push reaches the collector. > **Roadmap Note (Actor-Level Metrics):** The control plane latency, snapshot performance, and fleet utilization instruments above are in place. What remains is actor-level correlation: telemetry that an actor emits itself carries no Substrate-injected identity yet ([#853](https://github.com/agent-substrate/substrate/issues/853), [#761](https://github.com/agent-substrate/substrate/issues/761)). @@ -482,7 +482,7 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | Logs | pod stdout; the [actor events](#the-same-records-over-otlp), ateapi's lifecycle records and the ateworkers' usage samples, also to the collector's `debug` exporter | pod stdout. No OTLP logs: `OTEL_LOGS_EXPORTER` is unset | | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | -> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, and `atenet-egress` are pointed at the in-cluster collector, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally. The Kubernetes credential provider is the exception: it does not read the endpoint (see the [note on the Kind Prometheus](#2-metrics)). +> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider` are pointed at the in-cluster collector, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally. The Kubernetes credential provider also reads it, but its metrics are not in the Kind Prometheus UI (see the [note](#2-metrics)). > > Each of these components reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`. > diff --git a/manifests/ate-install/ate-otel-config.yaml b/manifests/ate-install/ate-otel-config.yaml index 17f5491061..1c2f679cc9 100644 --- a/manifests/ate-install/ate-otel-config.yaml +++ b/manifests/ate-install/ate-otel-config.yaml @@ -14,7 +14,10 @@ # Single source of truth for the OTLP exporter settings shared by every control # plane component (ate-api-server, ate-controller, atelet, atenet-router, -# atenet-egress, k8s-credential-provider), which consume it via `envFrom`. ate-controller additionally propagates these values +# atenet-egress, k8s-credential-provider), which consume it via `envFrom`. +# k8s-credential-provider exists only when the k8s.io credential provider is +# selected, and comes from manifests/egress-credential-injection, so restart it +# after editing this file. ate-controller additionally propagates these values # to the worker pods it creates, which have no other way to receive them, # and atenet-router's --otlp-collector-address defaults to the endpoint below, # so Envoy's own spans follow it too. The exporter selection is the exception: From abca19f02df783ad3730ed673f0acef93e7d98b5 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 6 Nov 2026 11:27:08 +0000 Subject: [PATCH 3/4] docs: fix the credential provider notes in observability Signed-off-by: Eric Curtin --- docs/guides/observability.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guides/observability.md b/docs/guides/observability.md index e1df2534a4..6b466a2626 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -430,7 +430,7 @@ To explore metrics locally: > **Note:** Storage is ephemeral (`emptyDir`), so metrics are lost when the Prometheus pod restarts. -> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics appear in neither: its pod carries no scrape annotation, so it is not in this UI. Its OTLP push reaches the collector. +> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics also reach the collector but not this UI, because its pod carries no scrape annotation. > **Roadmap Note (Actor-Level Metrics):** The control plane latency, snapshot performance, and fleet utilization instruments above are in place. What remains is actor-level correlation: telemetry that an actor emits itself carries no Substrate-injected identity yet ([#853](https://github.com/agent-substrate/substrate/issues/853), [#761](https://github.com/agent-substrate/substrate/issues/761)). @@ -484,7 +484,7 @@ To visualize traces locally: Telemetry is emitted over OTLP to your configured OpenTelemetry Collector. -> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The Kubernetes credential provider also reads it, but its metrics are not in the Kind Prometheus UI (see the [note on the Kind Prometheus](#2-metrics)). +> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The credential provider's metrics are not in the Kind Prometheus UI (see the [note on the Kind Prometheus](#2-metrics)). > > Each of these components reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`. > From ffcf8429a3c79cac7595f7b67df253a1e1adffce Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 13 Nov 2026 21:38:05 +0000 Subject: [PATCH 4/4] Review fixes: use the deployment constant, say the provider sends only metrics Signed-off-by: Eric Curtin --- cmd/ate-setup/internal/steps/overlay.go | 2 +- cmd/ate-setup/internal/steps/overlay_test.go | 4 ++-- docs/guides/observability.md | 2 +- manifests/ate-install/ate-otel-config.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index e2690bb29e..992408e5ef 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -612,7 +612,7 @@ const otelEndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT" // otelOverrideDeployments are the control plane Deployments that read // ate-otel-config. ate-controller additionally copies the values onto the // worker pods it creates, so one patch reaches the whole system. -var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress", "k8s-credential-provider"} +var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress", k8sCredentialProviderDeployment} // applyOtelEndpointOverride points all control plane telemetry at a different // collector for the duration of a measurement. See diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 3c507c2568..c71d2f6bcd 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -1436,7 +1436,7 @@ func TestApplyOtelEndpointOverride(t *testing.T) { provider := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: NamespaceAteSystem, - Name: "k8s-credential-provider", + Name: k8sCredentialProviderDeployment, }, } e := &Env{ @@ -1460,7 +1460,7 @@ func TestApplyOtelEndpointOverride(t *testing.T) { if !restartedAt(t, e, "deployment", "atenet-egress") { t.Error("atenet-egress was not restarted") } - if !restartedAt(t, e, "deployment", "k8s-credential-provider") { + if !restartedAt(t, e, "deployment", k8sCredentialProviderDeployment) { t.Error("k8s-credential-provider was not restarted") } if !restartedAt(t, e, "daemonset", "atelet") { diff --git a/docs/guides/observability.md b/docs/guides/observability.md index 6b466a2626..7911034d3c 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -484,7 +484,7 @@ To visualize traces locally: Telemetry is emitted over OTLP to your configured OpenTelemetry Collector. -> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The credential provider's metrics are not in the Kind Prometheus UI (see the [note on the Kind Prometheus](#2-metrics)). +> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The credential provider sends only metrics, so it is not among the sources of traces and OTLP logs. Its metrics are not in the Kind Prometheus UI (see the [note on the Kind Prometheus](#2-metrics)). > > Each of these components reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`. > diff --git a/manifests/ate-install/ate-otel-config.yaml b/manifests/ate-install/ate-otel-config.yaml index 1e94145432..9a36f6bff5 100644 --- a/manifests/ate-install/ate-otel-config.yaml +++ b/manifests/ate-install/ate-otel-config.yaml @@ -16,8 +16,8 @@ # plane component (ate-api-server, ate-controller, atelet, atenet-router, # atenet-egress, k8s-credential-provider), which consume it via `envFrom`. # k8s-credential-provider exists only when the k8s.io credential provider is -# selected, and comes from manifests/egress-credential-injection, so restart it -# after editing this file. ate-controller additionally propagates these values +# selected, and comes from manifests/egress-credential-injection, so a restart +# of the bundle workloads does not restart it. ate-controller additionally propagates these values # to the worker pods it creates, which have no other way to receive them, # and atenet-router's --otlp-collector-address defaults to the endpoint below, # so Envoy's own spans follow it too. The exporter selection is the exception: