Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions benchmarking/telemetry/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ go run ./cmd/ate-setup deploy ate-system \

`--otlp-endpoint` patches the `ate-otel-config` ConfigMap and restarts the
workloads that read it. One patch is sufficient for `ateapi`, `ate-controller`,
`atelet`, `atenet-router`, and `atenet-egress`, because each one reads the
ConfigMap through `envFrom`. `ate-controller` also copies the values to the
worker pods that it creates.
`atelet`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider`,
because each one reads the ConfigMap through `envFrom`. `ate-controller` also
copies the values to the worker pods that it creates.

The actor containers are different. Substrate puts no OTLP configuration in
them, thus they use the `env` of the ActorTemplate:
Expand Down
21 changes: 21 additions & 0 deletions cmd/ate-setup/internal/steps/credprovider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package steps

import (
"slices"
"testing"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand Down Expand Up @@ -46,6 +47,9 @@ func TestK8sCredentialProviderManifestsAgree(t *testing.T) {
if got := mountedConfigMaps(dep); len(got) != 1 || got[0] != k8sCredentialProviderPolicyConfigMap {
t.Errorf("deployment/%s mounts ConfigMaps %v, want only %q", k8sCredentialProviderDeployment, got, k8sCredentialProviderPolicyConfigMap)
}
if !slices.Contains(envFromConfigMaps(dep), otelConfigMap) {
t.Errorf("deployment/%s does not load configmap/%s through envFrom, so its metrics go to localhost:4317", k8sCredentialProviderDeployment, otelConfigMap)
}
svc := findObject(provider, "Service", k8sCredentialProviderDeployment)
if svc == nil {
t.Fatalf("provider manifest has no service/%s", k8sCredentialProviderDeployment)
Expand Down Expand Up @@ -202,3 +206,20 @@ func mountedConfigMaps(dep *unstructured.Unstructured) []string {
}
return names
}

// envFromConfigMaps returns the names of the ConfigMaps a Deployment's
// containers load through envFrom.
func envFromConfigMaps(dep *unstructured.Unstructured) []string {
containers, _, _ := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers")
var names []string
for _, c := range containers {
sources, _, _ := unstructured.NestedSlice(c.(map[string]any), "envFrom")
for _, src := range sources {
name, found, _ := unstructured.NestedString(src.(map[string]any), "configMapRef", "name")
if found {
names = append(names, name)
}
}
}
return names
}
2 changes: 1 addition & 1 deletion cmd/ate-setup/internal/steps/overlay.go
Original file line number Diff line number Diff line change
Expand Up @@ -612,7 +612,7 @@ const otelEndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT"
// otelOverrideDeployments are the control plane Deployments that read
// ate-otel-config. ate-controller additionally copies the values onto the
// worker pods it creates, so one patch reaches the whole system.
var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress"}
var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress", k8sCredentialProviderDeployment}

// applyOtelEndpointOverride points all control plane telemetry at a different
// collector for the duration of a measurement. See
Expand Down
11 changes: 10 additions & 1 deletion cmd/ate-setup/internal/steps/overlay_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1433,9 +1433,15 @@ func TestApplyOtelEndpointOverride(t *testing.T) {
Name: "atenet-egress",
},
}
provider := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Namespace: NamespaceAteSystem,
Name: k8sCredentialProviderDeployment,
},
}
e := &Env{
Cfg: &config.Config{OtlpEndpoint: endpoint},
Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), egress, atelet),
Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), egress, atelet, provider),
}

if err := e.applyOtelEndpointOverride(t.Context()); err != nil {
Expand All @@ -1454,6 +1460,9 @@ func TestApplyOtelEndpointOverride(t *testing.T) {
if !restartedAt(t, e, "deployment", "atenet-egress") {
t.Error("atenet-egress was not restarted")
}
if !restartedAt(t, e, "deployment", k8sCredentialProviderDeployment) {
t.Error("k8s-credential-provider was not restarted")
}
if !restartedAt(t, e, "daemonset", "atelet") {
t.Error("the atelet DaemonSet was not restarted")
}
Expand Down
4 changes: 2 additions & 2 deletions docs/guides/observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ To explore metrics locally:

> **Note:** Storage is ephemeral (`emptyDir`), so metrics are lost when the Prometheus pod restarts.

> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics appear in neither: its pod carries no scrape annotation and does not read `ate-otel-config`, so its OTLP push goes to the SDK default `localhost:4317` and fails.
> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics also reach the collector but not this UI, because its pod carries no scrape annotation.

> **Roadmap Note (Actor-Level Metrics):** The control plane latency, snapshot performance, and fleet utilization instruments above are in place. What remains is actor-level correlation: telemetry that an actor emits itself carries no Substrate-injected identity yet ([#853](https://github.com/agent-substrate/substrate/issues/853), [#761](https://github.com/agent-substrate/substrate/issues/761)).

Expand Down Expand Up @@ -484,7 +484,7 @@ To visualize traces locally:

Telemetry is emitted over OTLP to your configured OpenTelemetry Collector.

> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, and `atenet-egress` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The Kubernetes credential provider is the exception: it does not read the endpoint (see the [note on the Kind Prometheus](#2-metrics)).
> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, `atenet-egress`, and `k8s-credential-provider` are pointed at the in-cluster `opentelemetry-collector` in `otel-system`, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally (Prometheus exporter on `:8889`, Jaeger UI for traces, and the collector's `debug` exporter for OTLP logs). The credential provider sends only metrics, so it is not among the sources of traces and OTLP logs. Its metrics are not in the Kind Prometheus UI (see the [note on the Kind Prometheus](#2-metrics)).
>
> Each of these components reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`.
>
Expand Down
5 changes: 4 additions & 1 deletion manifests/ate-install/ate-otel-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@

# Single source of truth for the OTLP exporter settings shared by every control
# plane component (ate-api-server, ate-controller, atelet, atenet-router,
# atenet-egress), which consume it via `envFrom`. ate-controller additionally propagates these values
# atenet-egress, k8s-credential-provider), which consume it via `envFrom`.
# k8s-credential-provider exists only when the k8s.io credential provider is
# selected, and comes from manifests/egress-credential-injection, so a restart
# of the bundle workloads does not restart it. ate-controller additionally propagates these values
# to the worker pods it creates, which have no other way to receive them,
# and atenet-router's --otlp-collector-address defaults to the endpoint below,
# so Envoy's own spans follow it too. The exporter selection is the exception:
Expand Down
21 changes: 21 additions & 0 deletions manifests/egress-credential-injection/k8s-credential-provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,27 @@ spec:
# Enforce the atespace→namespace authorization policy (default-deny).
- "--namespace-policy-file=/etc/k8s-credential-provider/namespace-policy.yaml"
- "--log-level=info"
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: OTEL_RESOURCE_ATTRIBUTES
value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID)
# OTLP exporter settings shared by every control plane component; see
# manifests/ate-install/ate-otel-config.yaml. Without this the SDK
# falls back to localhost:4317 and every metric is lost.
envFrom:
Comment thread
ericcurtin marked this conversation as resolved.
Resolved
- configMapRef:
name: ate-otel-config
ports:
- name: grpc
containerPort: 50051
Expand Down
Loading