Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion benchmarking/atunnel/internal/workerhost/workerhost.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import (
"strconv"
"sync"

"go.opentelemetry.io/otel/metric/noop"

"go.ate.dev/substrate/internal/ateworker/actornet"
"go.ate.dev/substrate/internal/atunnel"
"go.ate.dev/substrate/internal/nodepath"
Expand Down Expand Up @@ -76,7 +78,7 @@ type actor struct {
// Start starts atunnel from cfg, as a worker's main does, and hosts actors
// actors whose egress tunnels to gatewayAddress.
func Start(ctx context.Context, cfg atunnel.Config, actors int, gatewayAddress string) (*Host, error) {
t, err := atunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream)
t, err := atunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream, noop.Meter{})
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/ateworker-gvisor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ func do(ctx context.Context) error {
}

actorLogger := actorlog.NewActorLogger(syncedWriter, metadata.OnGCE())
tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream)
tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream, mp.Meter(serviceName))
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/ateworker-microvm/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ func do(ctx context.Context) error {
return fmt.Errorf("while delegating the worker cgroup: %w", err)
}

tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream)
tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream, mp.Meter(serviceName))
if err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions docs/guides/observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo
| `ate.filecache.size` | atelet | up/down counter | bytes of disk a node-local file cache occupies, by `ate.filecache.name`, read on each collection; compare with `--sandbox-asset-cache-max-bytes` |
| `ate.filecache.evictions` | atelet | counter | entries the eviction passes removed from a node-local file cache, by `ate.filecache.name`; a steady rate at the budget means the cache thrashes and actors pay for downloads again |
| `ate.filecache.gc.passes` | atelet | counter | eviction passes over a node-local file cache, by `ate.filecache.name`, that had work to do, by outcome (`ate.filecache.gc.outcome`: `complete`, `shortfall`, `skipped`); a pass within budget is not counted |
| `ate.dns.queries` | ateworker-gvisor & ateworker-microvm | counter | actor DNS queries the sandbox DNS relay checked against the actor's egress policy, by `ate.dns.decision` (`allowed` or `denied`); a denied query gets `NXDOMAIN`, and most lookups also cause denials for the worker pod's search domains, which resolvers try first |

The table lists the OpenTelemetry instrument names. How a name appears in a query depends on the backend (Cloud Monitoring (GMP) / Kind collector).

Expand Down
58 changes: 55 additions & 3 deletions docs/metrics/registry/metrics.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,30 @@ groups:
value: skipped
brief: The pass could not size the cache and removed nothing. It retries on the next tick.

- id: registry.ate.dns
type: attribute_group
brief: >
The labels of the DNS relay of the sandbox. The relay checks each DNS
query of the actor against the egress policy of the actor.
attributes:
- id: ate.dns.decision
stability: development
brief: The decision of the egress policy on one DNS query.
type:
members:
- id: allowed
stability: development
value: allowed
brief: A rule of the policy allows the name. The relay sends the query to the resolvers of the worker pod.
- id: denied
stability: development
value: denied
brief: >
No rule allows the name, or the relay could not fetch the
policy and holds no earlier copy. The relay answers NXDOMAIN.
Thus the actor sees the same error as for a name that does not
exist.

- id: registry.ate.stats
type: attribute_group
brief: The labels of the resource measurements of the actors.
Expand Down Expand Up @@ -1598,6 +1622,35 @@ groups:
- ref: ate.router.outcome
requirement_level: required

# ---------------------------------------------------------------------------
# Metrics. ateworker
# ---------------------------------------------------------------------------

- id: metric.ate.dns.queries
type: metric
metric_name: ate.dns.queries
instrument: counter
unit: "{query}"
stability: development
brief: The number of actor DNS queries that the DNS relay of the sandbox checked against the egress policy, by decision.
note: >
The sandbox keeps the search domains of the worker pod and ndots:5. Thus
a stub resolver tries the search domains before most names, and most
lookups cause denials, even when a rule allows the name. Compare the
denied rate with its usual level. A rise means that egress policies block
names that actors look up. The actor sees only "bad address" or "could
not resolve host". The relay does not count a query that it drops or
rejects before the policy check, such as a malformed query.
annotations:
substrate:
emitted_by: [ateworker-gvisor, ateworker-microvm]
golden_signals: [traffic, errors]
code_anchor: internal/atunnel/dns/metrics.go
cuj: Actors cannot resolve external names. Does an egress policy deny them?
attributes:
- ref: ate.dns.decision
requirement_level: required

# ---------------------------------------------------------------------------
# Metrics. Upstream instrumentation
# ---------------------------------------------------------------------------
Expand All @@ -1611,9 +1664,8 @@ groups:
brief: The time, the rate and the errors of each gRPC method on the server.
note: >
The otelgrpc interceptor sends this instrument. The code of Substrate does
not make it. This instrument is the only metric of ateworker. ateworker has it on
the gRPC interface that atelet calls. credprovider is the
kubernetes-secrets credential provider in
not make it. ateworker has it on the gRPC interface that atelet calls.
credprovider is the kubernetes-secrets credential provider in
cmd/credential-provider/kubernetes-secrets.
annotations:
substrate:
Expand Down
8 changes: 8 additions & 0 deletions internal/ateattr/ateattr.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ const (
RouterResumeKey = attribute.Key("ate.router.resume")
RouterOutcomeKey = attribute.Key("ate.router.outcome")
StatsSourceKey = attribute.Key("ate.stats.source")
DNSDecisionKey = attribute.Key("ate.dns.decision")
)

// Keys of the actor usage record. Logs only: the measurements are unbounded and
Expand Down Expand Up @@ -239,6 +240,13 @@ const (
ImageCacheOutcomeTimeout = "timeout"
)

// Values for DNSDecisionKey: whether the actor's egress policy let the sandbox
// DNS relay forward a query. The relay answers a denied query with NXDOMAIN.
const (
DNSDecisionAllowed = "allowed"
DNSDecisionDenied = "denied"
)

// Values for FileCacheNameKey.
const (
// FileCacheSandboxAsset is the sandbox asset cache: the runsc binaries,
Expand Down
8 changes: 6 additions & 2 deletions internal/atunnel/dns/dnshandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,16 +61,18 @@ type dnsHandler struct {
pending *pendingRequests
limiter *limiter
allow queryPolicy
metrics *metrics
}

func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy) *dnsHandler {
func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy, metrics *metrics) *dnsHandler {
if allow == nil {
allow = func(dnsmessage.Question) bool { return true }
}
return &dnsHandler{
pending: pending,
limiter: lim,
allow: allow,
metrics: metrics,
}
}

Expand Down Expand Up @@ -122,7 +124,9 @@ func (h *dnsHandler) onRequestCommon(raw []byte) (dnsmessage.Header, dnsmessage.
// Note: NXDOMAIN seems to interact better with application stacks than other
// error codes such as REFUSED.
cq := canonicalizeQuestion(q)
if !h.allow(cq) {
allowed := h.allow(cq)
h.metrics.recordQuery(allowed)
if !allowed {
return hdr, dnsmessage.Question{}, action{
kind: actionReply,
payload: protocol.NXDomainPacket(hdr, q),
Expand Down
8 changes: 4 additions & 4 deletions internal/atunnel/dns/dnshandler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func TestDNSHandlerOnRequestDrop(t *testing.T) {
connections: &limiterSlots{max: 1},
}
pending := newPendingRequests(lim)
h := newDNSHandler(pending, lim, nil)
h := newDNSHandler(pending, lim, nil, nil)
upstreams := []string{"10.96.0.10:53"}

// 1. Short packet (< 12 bytes).
Expand Down Expand Up @@ -110,7 +110,7 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) {
allow := func(q dnsmessage.Question) bool {
return q.Name.String() != "blocked.example.com." && q.Type != dnsmessage.TypeTXT
}
h := newDNSHandler(pending, lim, allow)
h := newDNSHandler(pending, lim, allow, nil)
upstreams := []string{"10.96.0.10:53"}

qExample := dnsmessage.Question{
Expand Down Expand Up @@ -246,7 +246,7 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) {
func TestDNSHandlerUDPForwardAndResponseValidation(t *testing.T) {
lim := newLimiter()
pending := newPendingRequests(lim)
h := newDNSHandler(pending, lim, nil)
h := newDNSHandler(pending, lim, nil, nil)

upstream1 := &net.UDPAddr{IP: net.ParseIP("10.96.0.10"), Port: 53}
upstream2 := &net.UDPAddr{IP: net.ParseIP("10.96.0.11"), Port: 53}
Expand Down Expand Up @@ -358,7 +358,7 @@ func TestDNSHandlerUDPForwardAndResponseValidation(t *testing.T) {
func TestDNSHandlerTCPForwardAndResponseValidation(t *testing.T) {
lim := newLimiter()
pending := newPendingRequests(lim)
h := newDNSHandler(pending, lim, nil)
h := newDNSHandler(pending, lim, nil, nil)

query := dnsQuery(0xbeef)
act := h.onTCPRequest(bytes.Clone(query))
Expand Down
57 changes: 57 additions & 0 deletions internal/atunnel/dns/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright 2026 The Agent Substrate Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dns

import (
"context"
"fmt"

"go.opentelemetry.io/otel/metric"

"go.ate.dev/substrate/internal/ateattr"
)

const queriesMetric = "ate.dns.queries"

// metrics holds the DNS relay's metric instruments. A nil *metrics is a valid
// no-op, so call sites need no guard.
type metrics struct {
queries metric.Int64Counter
}

// newMetrics creates the DNS relay's metric instruments against meter.
func newMetrics(meter metric.Meter) (*metrics, error) {
queries, err := meter.Int64Counter(
queriesMetric,
metric.WithUnit("{query}"),
metric.WithDescription("Number of actor DNS queries the sandbox DNS relay checked against the actor's egress policy, by decision."),
)
if err != nil {
return nil, fmt.Errorf("create %s counter: %w", queriesMetric, err)
}
return &metrics{queries: queries}, nil
}

// recordQuery counts one query the egress policy allowed or denied.
func (m *metrics) recordQuery(allowed bool) {
if m == nil {
return
}
decision := ateattr.DNSDecisionDenied
if allowed {
decision = ateattr.DNSDecisionAllowed
}
m.queries.Add(context.Background(), 1, metric.WithAttributes(ateattr.DNSDecisionKey.String(decision)))
}
122 changes: 122 additions & 0 deletions internal/atunnel/dns/metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Copyright 2026 The Agent Substrate Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dns

import (
"context"
"os"
"path/filepath"
"testing"

"github.com/google/go-cmp/cmp"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"golang.org/x/net/dns/dnsmessage"

"go.ate.dev/substrate/internal/ateattr"
)

// collectQueries returns the ate.dns.queries count for each ate.dns.decision.
func collectQueries(t *testing.T, reader *sdkmetric.ManualReader) map[string]int64 {
t.Helper()
var rm metricdata.ResourceMetrics
if err := reader.Collect(context.Background(), &rm); err != nil {
t.Fatalf("collect: %v", err)
}
got := make(map[string]int64)
for _, sm := range rm.ScopeMetrics {
for _, m := range sm.Metrics {
if m.Name != queriesMetric {
continue
}
if m.Unit != "{query}" {
t.Errorf("unit = %q, want {query}", m.Unit)
}
sum, ok := m.Data.(metricdata.Sum[int64])
if !ok {
t.Fatalf("data type = %T, want Sum[int64]", m.Data)
}
if !sum.IsMonotonic {
t.Error("IsMonotonic = false, want true")
}
for _, dp := range sum.DataPoints {
if dp.Attributes.Len() != 1 {
t.Errorf("attributes = %v, want only %s", dp.Attributes.ToSlice(), ateattr.DNSDecisionKey)
}
v, _ := dp.Attributes.Value(ateattr.DNSDecisionKey)
got[v.AsString()] += dp.Value
}
}
}
return got
}

// TestQueriesCountsPolicyDecisions: every query the egress policy judges is
// counted once, on either transport. A malformed query never reaches the
// policy and is not counted.
func TestQueriesCountsPolicyDecisions(t *testing.T) {
reader := sdkmetric.NewManualReader()
m, err := newMetrics(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor"))
if err != nil {
t.Fatalf("newMetrics: %v", err)
}
lim := newLimiter()
allow := func(q dnsmessage.Question) bool { return q.Name.String() != "blocked.example.com." }
h := newDNSHandler(newPendingRequests(lim), lim, allow, m)
upstreams := []string{"10.96.0.10:53"}

allowed, _ := dnsQueryForName(t, 1, "example.com.")
denied, _ := dnsQueryForName(t, 2, "blocked.example.com.")
h.onUDPRequest(allowed, nil, upstreams)
h.onTCPRequest(allowed)
h.onUDPRequest(denied, nil, upstreams)
h.onTCPRequest(denied)
h.onTCPRequest(buildTestMessage(t, dnsmessage.Header{ID: 3}, nil))

want := map[string]int64{
ateattr.DNSDecisionAllowed: 2,
ateattr.DNSDecisionDenied: 2,
}
if diff := cmp.Diff(want, collectQueries(t, reader)); diff != "" {
t.Errorf("ate.dns.queries mismatch (-want +got):\n%s", diff)
}
}

// TestNewRelayCountsQueries: a relay from NewRelay counts on its meter.
func TestNewRelayCountsQueries(t *testing.T) {
path := filepath.Join(t.TempDir(), "resolv.conf")
if err := os.WriteFile(path, []byte("nameserver 127.0.0.1\n"), 0o644); err != nil {
t.Fatal(err)
}
reader := sdkmetric.NewManualReader()
relay, err := NewRelay(path, sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor"))
if err != nil {
t.Fatalf("NewRelay: %v", err)
}
// The actor has no policy, so the relay answers the query itself.
_, udpAddr, _ := serveLoopbackWithPolicy(t, relay, staticPolicy{})
conn := dialRelay(t, "udp", udpAddr)

query, q := dnsQueryForName(t, 1, "example.com.")
if _, err := conn.Write(query); err != nil {
t.Fatal(err)
}
checkAnswerForQuestion(t, readWithin(t, conn), 1, q, dnsmessage.RCodeNameError)

want := map[string]int64{ateattr.DNSDecisionDenied: 1}
if diff := cmp.Diff(want, collectQueries(t, reader)); diff != "" {
t.Errorf("ate.dns.queries mismatch (-want +got):\n%s", diff)
}
}
1 change: 1 addition & 0 deletions internal/atunnel/dns/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,4 @@ UDP and TCP call up to the DNS protocol layer (`golang.org/x/net/dns/dnsmessage`
- `evaluator` (`evaluator.go`) compiles the hostname and wildcard patterns from an `EgressPolicy`'s `http`, `https`, and `tls_passthrough` rules and evaluates normalized `QNAME`s via `allowsHostname(qname)`.
- `egressPolicyCheck` adapts the source to the `dnsHandler`'s `queryPolicy` hook, caching the compiled `*evaluator` for the current `*ateletpb.EgressPolicy` pointer and consulting it for every query. Fetching uses the `Server`'s context, so `Stop()` cancels a fetch in progress.
- `allowQuery` evaluates a single `dnsmessage.Question` against a policy using `evaluator`.
- `metrics` (`metrics.go`) counts each decision on `ate.dns.queries`, by `ate.dns.decision`.
Loading
Loading