diff --git a/benchmarking/telemetry/README.md b/benchmarking/telemetry/README.md index c8492c712c..89f18b9e32 100644 --- a/benchmarking/telemetry/README.md +++ b/benchmarking/telemetry/README.md @@ -46,9 +46,9 @@ go run ./cmd/ate-setup deploy ate-system \ `--otlp-endpoint` patches the `ate-otel-config` ConfigMap and restarts the workloads that read it. One patch is sufficient for `ateapi`, `ate-controller`, -`atelet`, `atenet-router`, and `atenet-egress`, because each one reads the -ConfigMap through `envFrom`. `ate-controller` also copies the values to the -worker pods that it creates. +`atelet`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider`, +because each one reads the ConfigMap through `envFrom`. `ate-controller` also +copies the values to the worker pods that it creates. The actor containers are different. Substrate puts no OTLP configuration in them, thus they use the `env` of the ActorTemplate: diff --git a/cmd/ate-setup/internal/steps/credprovider_test.go b/cmd/ate-setup/internal/steps/credprovider_test.go index 06edc380a7..b797e8ca51 100644 --- a/cmd/ate-setup/internal/steps/credprovider_test.go +++ b/cmd/ate-setup/internal/steps/credprovider_test.go @@ -15,6 +15,7 @@ package steps import ( + "slices" "testing" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -46,6 +47,9 @@ func TestK8sCredentialProviderManifestsAgree(t *testing.T) { if got := mountedConfigMaps(dep); len(got) != 1 || got[0] != k8sCredentialProviderPolicyConfigMap { t.Errorf("deployment/%s mounts ConfigMaps %v, want only %q", k8sCredentialProviderDeployment, got, k8sCredentialProviderPolicyConfigMap) } + if !slices.Contains(envFromConfigMaps(dep), otelConfigMap) { + t.Errorf("deployment/%s does not load configmap/%s through envFrom, so its metrics go to localhost:4317", k8sCredentialProviderDeployment, otelConfigMap) + } svc := findObject(provider, "Service", k8sCredentialProviderDeployment) if svc == nil { t.Fatalf("provider manifest has no service/%s", k8sCredentialProviderDeployment) @@ -202,3 +206,20 @@ func mountedConfigMaps(dep *unstructured.Unstructured) []string { } return names } + +// envFromConfigMaps returns the names of the ConfigMaps a Deployment's +// containers load through envFrom. +func envFromConfigMaps(dep *unstructured.Unstructured) []string { + containers, _, _ := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") + var names []string + for _, c := range containers { + sources, _, _ := unstructured.NestedSlice(c.(map[string]any), "envFrom") + for _, src := range sources { + name, found, _ := unstructured.NestedString(src.(map[string]any), "configMapRef", "name") + if found { + names = append(names, name) + } + } + } + return names +} diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 2fbaf8ca66..992408e5ef 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -612,7 +612,7 @@ const otelEndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT" // otelOverrideDeployments are the control plane Deployments that read // ate-otel-config. ate-controller additionally copies the values onto the // worker pods it creates, so one patch reaches the whole system. -var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress"} +var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress", k8sCredentialProviderDeployment} // applyOtelEndpointOverride points all control plane telemetry at a different // collector for the duration of a measurement. See diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index ab735825d3..c71d2f6bcd 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -1433,9 +1433,15 @@ func TestApplyOtelEndpointOverride(t *testing.T) { Name: "atenet-egress", }, } + provider := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: NamespaceAteSystem, + Name: k8sCredentialProviderDeployment, + }, + } e := &Env{ Cfg: &config.Config{OtlpEndpoint: endpoint}, - Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), egress, atelet), + Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), egress, atelet, provider), } if err := e.applyOtelEndpointOverride(t.Context()); err != nil { @@ -1454,6 +1460,9 @@ func TestApplyOtelEndpointOverride(t *testing.T) { if !restartedAt(t, e, "deployment", "atenet-egress") { t.Error("atenet-egress was not restarted") } + if !restartedAt(t, e, "deployment", k8sCredentialProviderDeployment) { + t.Error("k8s-credential-provider was not restarted") + } if !restartedAt(t, e, "daemonset", "atelet") { t.Error("the atelet DaemonSet was not restarted") } diff --git a/docs/guides/observability.md b/docs/guides/observability.md index 8969b174fe..7911034d3c 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -430,7 +430,7 @@ To explore metrics locally: > **Note:** Storage is ephemeral (`emptyDir`), so metrics are lost when the Prometheus pod restarts. -> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics appear in neither: its pod carries no scrape annotation and does not read `ate-otel-config`, so its OTLP push goes to the SDK default `localhost:4317` and fails. +> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics also reach the collector but not this UI, because its pod carries no scrape annotation. > **Roadmap Note (Actor-Level Metrics):** The control plane latency, snapshot performance, and fleet utilization instruments above are in place. What remains is actor-level correlation: telemetry that an actor emits itself carries no Substrate-injected identity yet ([#853](https://github.com/agent-substrate/substrate/issues/853), [#761](https://github.com/agent-substrate/substrate/issues/761)). @@ -484,7 +484,7 @@ To visualize traces locally: Telemetry is emitted over OTLP to your configured OpenTelemetry Collector. -> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, and `atenet-egress` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The Kubernetes credential provider is the exception: it does not read the endpoint (see the [note on the Kind Prometheus](#2-metrics)). +> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The credential provider sends only metrics, so it is not among the sources of traces and OTLP logs. Its metrics are not in the Kind Prometheus UI (see the [note on the Kind Prometheus](#2-metrics)). > > Each of these components reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`. > diff --git a/manifests/ate-install/ate-otel-config.yaml b/manifests/ate-install/ate-otel-config.yaml index ac0f3e2315..9a36f6bff5 100644 --- a/manifests/ate-install/ate-otel-config.yaml +++ b/manifests/ate-install/ate-otel-config.yaml @@ -14,7 +14,10 @@ # Single source of truth for the OTLP exporter settings shared by every control # plane component (ate-api-server, ate-controller, atelet, atenet-router, -# atenet-egress), which consume it via `envFrom`. ate-controller additionally propagates these values +# atenet-egress, k8s-credential-provider), which consume it via `envFrom`. +# k8s-credential-provider exists only when the k8s.io credential provider is +# selected, and comes from manifests/egress-credential-injection, so a restart +# of the bundle workloads does not restart it. ate-controller additionally propagates these values # to the worker pods it creates, which have no other way to receive them, # and atenet-router's --otlp-collector-address defaults to the endpoint below, # so Envoy's own spans follow it too. The exporter selection is the exception: diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml index 628d903774..8c72302d0c 100644 --- a/manifests/egress-credential-injection/k8s-credential-provider.yaml +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -89,6 +89,27 @@ spec: # Enforce the atespace→namespace authorization policy (default-deny). - "--namespace-policy-file=/etc/k8s-credential-provider/namespace-policy.yaml" - "--log-level=info" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) + # OTLP exporter settings shared by every control plane component; see + # manifests/ate-install/ate-otel-config.yaml. Without this the SDK + # falls back to localhost:4317 and every metric is lost. + envFrom: + - configMapRef: + name: ate-otel-config ports: - name: grpc containerPort: 50051