From cee55c39aa2a3ac513ee47b35a86c34c4e650360 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Wed, 28 Oct 2026 11:16:18 -0700 Subject: [PATCH 1/3] Count actor DNS queries by egress policy decision Signed-off-by: Lior Lieberman --- .../atunnel/internal/workerhost/workerhost.go | 4 +- cmd/ateworker-gvisor/main.go | 2 +- cmd/ateworker-microvm/main.go | 2 +- docs/guides/observability.md | 1 + docs/metrics/registry/metrics.yaml | 64 +++++++- internal/ateattr/ateattr.go | 9 ++ internal/atunnel/dns/dnshandler.go | 20 ++- internal/atunnel/dns/dnshandler_test.go | 8 +- internal/atunnel/dns/metrics.go | 78 ++++++++++ internal/atunnel/dns/metrics_test.go | 139 ++++++++++++++++++ internal/atunnel/dns/readme.md | 1 + internal/atunnel/dns/relay.go | 20 ++- internal/atunnel/dns/relay_test.go | 3 +- internal/atunnel/dns/resolvconf.go | 22 +-- internal/atunnel/dns/resolvconf_test.go | 33 +++-- internal/atunnel/dns/server.go | 4 +- internal/atunnel/dns/tcphandler_test.go | 2 +- internal/atunnel/dns/udphandler_test.go | 2 +- internal/atunnel/tunnel.go | 8 +- internal/atunnel/tunnel_test.go | 7 +- 20 files changed, 377 insertions(+), 52 deletions(-) create mode 100644 internal/atunnel/dns/metrics.go create mode 100644 internal/atunnel/dns/metrics_test.go diff --git a/benchmarking/atunnel/internal/workerhost/workerhost.go b/benchmarking/atunnel/internal/workerhost/workerhost.go index 232e0f4499..a29ae8e2bb 100644 --- a/benchmarking/atunnel/internal/workerhost/workerhost.go +++ b/benchmarking/atunnel/internal/workerhost/workerhost.go @@ -29,6 +29,8 @@ import ( "strconv" "sync" + "go.opentelemetry.io/otel/metric/noop" + "go.ate.dev/substrate/internal/ateworker/actornet" "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/nodepath" @@ -76,7 +78,7 @@ type actor struct { // Start starts atunnel from cfg, as a worker's main does, and hosts actors // actors whose egress tunnels to gatewayAddress. func Start(ctx context.Context, cfg atunnel.Config, actors int, gatewayAddress string) (*Host, error) { - t, err := atunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream) + t, err := atunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream, noop.Meter{}) if err != nil { return nil, err } diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index d42598a5cd..75905878fa 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -263,7 +263,7 @@ func do(ctx context.Context) error { } actorLogger := actorlog.NewActorLogger(syncedWriter, metadata.OnGCE()) - tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream) + tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream, mp.Meter(serviceName)) if err != nil { return err } diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index 845f26d006..8fba5830f9 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -234,7 +234,7 @@ func do(ctx context.Context) error { return fmt.Errorf("while delegating the worker cgroup: %w", err) } - tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream) + tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream, mp.Meter(serviceName)) if err != nil { return err } diff --git a/docs/guides/observability.md b/docs/guides/observability.md index 8969b174fe..c248eb741e 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -305,6 +305,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.filecache.size` | atelet | up/down counter | bytes of disk a node-local file cache occupies, by `ate.filecache.name`, read on each collection; compare with `--sandbox-asset-cache-max-bytes` | | `ate.filecache.evictions` | atelet | counter | entries the eviction passes removed from a node-local file cache, by `ate.filecache.name`; a steady rate at the budget means the cache thrashes and actors pay for downloads again | | `ate.filecache.gc.passes` | atelet | counter | eviction passes over a node-local file cache, by `ate.filecache.name`, that had work to do, by outcome (`ate.filecache.gc.outcome`: `complete`, `shortfall`, `skipped`); a pass within budget is not counted | +| `ate.dns.queries` | ateworker-gvisor & ateworker-microvm | counter | actor DNS queries the sandbox DNS relay checked against the actor's egress policy, by `ate.dns.decision`: `allowed`, `denied` (answered `NXDOMAIN`), or `denied_search_domain` (a denied name under the worker pod's search domains, which resolvers try before most names) | The table lists the OpenTelemetry instrument names. How a name appears in a query depends on the backend (Cloud Monitoring (GMP) / Kind collector). diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 93d4214b07..6ed1e29e0b 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -634,6 +634,39 @@ groups: value: skipped brief: The pass could not size the cache and removed nothing. It retries on the next tick. + - id: registry.ate.dns + type: attribute_group + brief: > + The labels of the DNS relay of the sandbox. The relay checks each DNS + query of the actor against the egress policy of the actor. + attributes: + - id: ate.dns.decision + stability: development + brief: The decision of the egress policy on one DNS query. + type: + members: + - id: allowed + stability: development + value: allowed + brief: A rule of the policy allows the name. The relay sends the query to the resolvers of the worker pod. + - id: denied + stability: development + value: denied + brief: > + No rule allows the name, or the relay could not fetch the + policy and holds no earlier copy. The relay answers NXDOMAIN. + Thus the actor sees the same error as for a name that does not + exist. + - id: denied_search_domain + stability: development + value: denied_search_domain + brief: > + The same as denied, for a name under one of the search domains + of the worker pod. The sandbox keeps those search domains and + ndots:5. Thus a stub resolver tries them before most names, and + most lookups cause these denials. A denied cluster name, such as + a service in another namespace, is also counted here. + - id: registry.ate.stats type: attribute_group brief: The labels of the resource measurements of the actors. @@ -1598,6 +1631,32 @@ groups: - ref: ate.router.outcome requirement_level: required + # --------------------------------------------------------------------------- + # Metrics. ateworker + # --------------------------------------------------------------------------- + + - id: metric.ate.dns.queries + type: metric + metric_name: ate.dns.queries + instrument: counter + unit: "{query}" + stability: development + brief: The number of actor DNS queries that the DNS relay of the sandbox checked against the egress policy, by decision. + note: > + A denied rate above zero means that egress policies block external names + that actors look up. The actor sees only "bad address" or "could not + resolve host". The relay does not count a query that it drops or rejects + before the policy check, such as a malformed query. + annotations: + substrate: + emitted_by: [ateworker-gvisor, ateworker-microvm] + golden_signals: [traffic, errors] + code_anchor: internal/atunnel/dns/metrics.go + cuj: Actors cannot resolve external names. Does an egress policy deny them? + attributes: + - ref: ate.dns.decision + requirement_level: required + # --------------------------------------------------------------------------- # Metrics. Upstream instrumentation # --------------------------------------------------------------------------- @@ -1611,9 +1670,8 @@ groups: brief: The time, the rate and the errors of each gRPC method on the server. note: > The otelgrpc interceptor sends this instrument. The code of Substrate does - not make it. This instrument is the only metric of ateworker. ateworker has it on - the gRPC interface that atelet calls. credprovider is the - kubernetes-secrets credential provider in + not make it. ateworker has it on the gRPC interface that atelet calls. + credprovider is the kubernetes-secrets credential provider in cmd/credential-provider/kubernetes-secrets. annotations: substrate: diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 5f55a8b2c3..e2f13da10e 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -167,6 +167,7 @@ const ( RouterResumeKey = attribute.Key("ate.router.resume") RouterOutcomeKey = attribute.Key("ate.router.outcome") StatsSourceKey = attribute.Key("ate.stats.source") + DNSDecisionKey = attribute.Key("ate.dns.decision") ) // Keys of the actor usage record. Logs only: the measurements are unbounded and @@ -239,6 +240,14 @@ const ( ImageCacheOutcomeTimeout = "timeout" ) +// Values for DNSDecisionKey: whether the actor's egress policy let the sandbox +// DNS relay forward a query. The relay answers a denied query with NXDOMAIN. +const ( + DNSDecisionAllowed = "allowed" + DNSDecisionDenied = "denied" + DNSDecisionDeniedSearchDomain = "denied_search_domain" +) + // Values for FileCacheNameKey. const ( // FileCacheSandboxAsset is the sandbox asset cache: the runsc binaries, diff --git a/internal/atunnel/dns/dnshandler.go b/internal/atunnel/dns/dnshandler.go index ae19e99c7e..c3b22e5f15 100644 --- a/internal/atunnel/dns/dnshandler.go +++ b/internal/atunnel/dns/dnshandler.go @@ -58,19 +58,21 @@ type queryPolicy func(q dnsmessage.Question) bool // dnsHandler inspects DNS requests and responses and returns transport-agnostic // routing decisions. type dnsHandler struct { - pending *pendingRequests - limiter *limiter - allow queryPolicy + pending *pendingRequests + limiter *limiter + allow queryPolicy + instruments *instruments } -func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy) *dnsHandler { +func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy, instruments *instruments) *dnsHandler { if allow == nil { allow = func(dnsmessage.Question) bool { return true } } return &dnsHandler{ - pending: pending, - limiter: lim, - allow: allow, + pending: pending, + limiter: lim, + allow: allow, + instruments: instruments, } } @@ -122,7 +124,9 @@ func (h *dnsHandler) onRequestCommon(raw []byte) (dnsmessage.Header, dnsmessage. // Note: NXDOMAIN seems to interact better with application stacks than other // error codes such as REFUSED. cq := canonicalizeQuestion(q) - if !h.allow(cq) { + allowed := h.allow(cq) + h.instruments.recordQuery(cq, allowed) + if !allowed { return hdr, dnsmessage.Question{}, action{ kind: actionReply, payload: protocol.NXDomainPacket(hdr, q), diff --git a/internal/atunnel/dns/dnshandler_test.go b/internal/atunnel/dns/dnshandler_test.go index f07061b9da..a4480c7774 100644 --- a/internal/atunnel/dns/dnshandler_test.go +++ b/internal/atunnel/dns/dnshandler_test.go @@ -76,7 +76,7 @@ func TestDNSHandlerOnRequestDrop(t *testing.T) { connections: &limiterSlots{max: 1}, } pending := newPendingRequests(lim) - h := newDNSHandler(pending, lim, nil) + h := newDNSHandler(pending, lim, nil, nil) upstreams := []string{"10.96.0.10:53"} // 1. Short packet (< 12 bytes). @@ -110,7 +110,7 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { allow := func(q dnsmessage.Question) bool { return q.Name.String() != "blocked.example.com." && q.Type != dnsmessage.TypeTXT } - h := newDNSHandler(pending, lim, allow) + h := newDNSHandler(pending, lim, allow, nil) upstreams := []string{"10.96.0.10:53"} qExample := dnsmessage.Question{ @@ -246,7 +246,7 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { func TestDNSHandlerUDPForwardAndResponseValidation(t *testing.T) { lim := newLimiter() pending := newPendingRequests(lim) - h := newDNSHandler(pending, lim, nil) + h := newDNSHandler(pending, lim, nil, nil) upstream1 := &net.UDPAddr{IP: net.ParseIP("10.96.0.10"), Port: 53} upstream2 := &net.UDPAddr{IP: net.ParseIP("10.96.0.11"), Port: 53} @@ -358,7 +358,7 @@ func TestDNSHandlerUDPForwardAndResponseValidation(t *testing.T) { func TestDNSHandlerTCPForwardAndResponseValidation(t *testing.T) { lim := newLimiter() pending := newPendingRequests(lim) - h := newDNSHandler(pending, lim, nil) + h := newDNSHandler(pending, lim, nil, nil) query := dnsQuery(0xbeef) act := h.onTCPRequest(bytes.Clone(query)) diff --git a/internal/atunnel/dns/metrics.go b/internal/atunnel/dns/metrics.go new file mode 100644 index 0000000000..9a2e06a052 --- /dev/null +++ b/internal/atunnel/dns/metrics.go @@ -0,0 +1,78 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "fmt" + "strings" + + "go.opentelemetry.io/otel/metric" + "golang.org/x/net/dns/dnsmessage" + + "go.ate.dev/substrate/internal/ateattr" +) + +const queriesMetric = "ate.dns.queries" + +// instruments holds the DNS relay's instruments. A nil *instruments is a valid +// no-op, so call sites need no guard. +type instruments struct { + queries metric.Int64Counter + // searchSuffixes are the worker pod's search domains as ".domain.", in + // lowercase. + searchSuffixes []string +} + +// newInstruments creates the DNS relay's instruments against meter. search +// holds the search domains of the worker pod, which the sandbox's resolv.conf +// keeps. +func newInstruments(meter metric.Meter, search []string) (*instruments, error) { + queries, err := meter.Int64Counter( + queriesMetric, + metric.WithUnit("{query}"), + metric.WithDescription("Number of actor DNS queries the sandbox DNS relay checked against the actor's egress policy, by decision."), + ) + if err != nil { + return nil, fmt.Errorf("create %s counter: %w", queriesMetric, err) + } + i := &instruments{queries: queries} + for _, d := range search { + i.searchSuffixes = append(i.searchSuffixes, "."+strings.ToLower(strings.TrimSuffix(d, "."))+".") + } + return i, nil +} + +// recordQuery counts one query the egress policy allowed or denied. q is +// canonical, so its name is in lowercase. +func (i *instruments) recordQuery(q dnsmessage.Question, allowed bool) { + if i == nil { + return + } + decision := ateattr.DNSDecisionAllowed + if !allowed { + decision = ateattr.DNSDecisionDenied + // A stub resolver tries the search domains before most names (ndots:5 + // in Kubernetes), so most lookups cause these denials. + name := q.Name.String() + for _, suffix := range i.searchSuffixes { + if strings.HasSuffix(name, suffix) { + decision = ateattr.DNSDecisionDeniedSearchDomain + break + } + } + } + i.queries.Add(context.Background(), 1, metric.WithAttributes(ateattr.DNSDecisionKey.String(decision))) +} diff --git a/internal/atunnel/dns/metrics_test.go b/internal/atunnel/dns/metrics_test.go new file mode 100644 index 0000000000..59c0647b03 --- /dev/null +++ b/internal/atunnel/dns/metrics_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "golang.org/x/net/dns/dnsmessage" + + "go.ate.dev/substrate/internal/ateattr" +) + +// collectQueries returns the ate.dns.queries count for each ate.dns.decision. +func collectQueries(t *testing.T, reader *sdkmetric.ManualReader) map[string]int64 { + t.Helper() + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("collect: %v", err) + } + got := make(map[string]int64) + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != queriesMetric { + continue + } + if m.Unit != "{query}" { + t.Errorf("unit = %q, want {query}", m.Unit) + } + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + t.Fatalf("data type = %T, want Sum[int64]", m.Data) + } + if !sum.IsMonotonic { + t.Error("IsMonotonic = false, want true") + } + for _, dp := range sum.DataPoints { + if dp.Attributes.Len() != 1 { + t.Errorf("attributes = %v, want only %s", dp.Attributes.ToSlice(), ateattr.DNSDecisionKey) + } + v, _ := dp.Attributes.Value(ateattr.DNSDecisionKey) + got[v.AsString()] += dp.Value + } + } + } + return got +} + +// TestQueriesCountsPolicyDecisions: every query the egress policy judges is +// counted once, on either transport. A malformed query never reaches the +// policy and is not counted. +func TestQueriesCountsPolicyDecisions(t *testing.T) { + reader := sdkmetric.NewManualReader() + meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor") + inst, err := newInstruments(meter, []string{"ate-workers.svc.cluster.local", "SVC.Cluster.Local."}) + if err != nil { + t.Fatalf("newInstruments: %v", err) + } + lim := newLimiter() + allow := func(q dnsmessage.Question) bool { return !strings.HasPrefix(q.Name.String(), "blocked.") } + h := newDNSHandler(newPendingRequests(lim), lim, allow, inst) + + for i, name := range []string{ + "example.com.", + "example.com.svc.cluster.local.", + "blocked.example.com.", + "blocked.notsvc.cluster.local.", + "blocked.example.com.ate-workers.svc.cluster.local.", + "Blocked.Example.COM.svc.CLUSTER.local.", + } { + raw, _ := dnsQueryForName(t, uint16(i), name) + h.onTCPRequest(raw) + } + allowed, _ := dnsQueryForName(t, 100, "example.com.") + h.onUDPRequest(allowed, nil, []string{"10.96.0.10:53"}) + denied, _ := dnsQueryForName(t, 101, "blocked.example.com.") + h.onUDPRequest(denied, nil, []string{"10.96.0.10:53"}) + h.onTCPRequest(buildTestMessage(t, dnsmessage.Header{ID: 102}, nil)) + + want := map[string]int64{ + ateattr.DNSDecisionAllowed: 3, + ateattr.DNSDecisionDenied: 3, + ateattr.DNSDecisionDeniedSearchDomain: 2, + } + if diff := cmp.Diff(want, collectQueries(t, reader)); diff != "" { + t.Errorf("ate.dns.queries mismatch (-want +got):\n%s", diff) + } +} + +// TestNewRelayCountsQueries: a relay from NewRelay counts on its meter and +// takes the search domains from the resolv.conf it reads. +func TestNewRelayCountsQueries(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + if err := os.WriteFile(path, []byte("search ate-workers.svc.cluster.local\nnameserver 127.0.0.1\n"), 0o644); err != nil { + t.Fatal(err) + } + reader := sdkmetric.NewManualReader() + relay, err := NewRelay(path, sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor")) + if err != nil { + t.Fatalf("NewRelay: %v", err) + } + // The actor has no policy, so the relay answers every query itself. + _, udpAddr, _ := serveLoopbackWithPolicy(t, relay, staticPolicy{}) + conn := dialRelay(t, "udp", udpAddr) + + for i, name := range []string{"example.com.", "example.com.ate-workers.svc.cluster.local."} { + query, q := dnsQueryForName(t, uint16(i), name) + if _, err := conn.Write(query); err != nil { + t.Fatal(err) + } + checkAnswerForQuestion(t, readWithin(t, conn), uint16(i), q, dnsmessage.RCodeNameError) + } + + want := map[string]int64{ + ateattr.DNSDecisionDenied: 1, + ateattr.DNSDecisionDeniedSearchDomain: 1, + } + if diff := cmp.Diff(want, collectQueries(t, reader)); diff != "" { + t.Errorf("ate.dns.queries mismatch (-want +got):\n%s", diff) + } +} diff --git a/internal/atunnel/dns/readme.md b/internal/atunnel/dns/readme.md index 2b5bda0010..ef9e0befd8 100644 --- a/internal/atunnel/dns/readme.md +++ b/internal/atunnel/dns/readme.md @@ -73,3 +73,4 @@ UDP and TCP call up to the DNS protocol layer (`golang.org/x/net/dns/dnsmessage` - `evaluator` (`evaluator.go`) compiles the hostname and wildcard patterns from an `EgressPolicy`'s `http`, `https`, and `tls_passthrough` rules and evaluates normalized `QNAME`s via `allowsHostname(qname)`. - `egressPolicyCheck` adapts the source to the `dnsHandler`'s `queryPolicy` hook, caching the compiled `*evaluator` for the current `*ateletpb.EgressPolicy` pointer and consulting it for every query. Fetching uses the `Server`'s context, so `Stop()` cancels a fetch in progress. - `allowQuery` evaluates a single `dnsmessage.Question` against a policy using `evaluator`. +- `instruments` (`metrics.go`) counts each decision on `ate.dns.queries`, by `ate.dns.decision`. diff --git a/internal/atunnel/dns/relay.go b/internal/atunnel/dns/relay.go index 71f215cf08..348d653935 100644 --- a/internal/atunnel/dns/relay.go +++ b/internal/atunnel/dns/relay.go @@ -21,6 +21,8 @@ import ( "net" "net/netip" "time" + + "go.opentelemetry.io/otel/metric" ) const ( @@ -39,15 +41,25 @@ type Relay struct { // dialer reaches upstream resolvers from the worker namespace. dialer *net.Dialer + + instruments *instruments } // NewRelay reads nameservers from resolvConfPath and forwards to each on port 53. -func NewRelay(resolvConfPath string) (*Relay, error) { - upstreams, err := resolvConfNameservers(resolvConfPath) +// It reports its metrics on meter. +func NewRelay(resolvConfPath string, meter metric.Meter) (*Relay, error) { + upstreams, search, err := readResolvConf(resolvConfPath) if err != nil { return nil, err } - return NewRelayForUpstreams(upstreams) + r, err := NewRelayForUpstreams(upstreams) + if err != nil { + return nil, err + } + if r.instruments, err = newInstruments(meter, search); err != nil { + return nil, err + } + return r, nil } func normalizeAddr(addr net.Addr) string { @@ -98,5 +110,5 @@ func NewRelayForUpstreams(upstreams []string) (*Relay, error) { // serveOn serves DNS on netC's sockets, which the caller has already bound, // and dials upstreams with netC's dialer. func (r *Relay) serveOn(ctx context.Context, netC *netConn, policy PolicySource) (*Server, error) { - return newServer(ctx, &serverConfig{upstreams: r.upstreams, policy: policy}, netC) + return newServer(ctx, &serverConfig{upstreams: r.upstreams, policy: policy, instruments: r.instruments}, netC) } diff --git a/internal/atunnel/dns/relay_test.go b/internal/atunnel/dns/relay_test.go index 2ba5b78eb6..8186b98b92 100644 --- a/internal/atunnel/dns/relay_test.go +++ b/internal/atunnel/dns/relay_test.go @@ -28,6 +28,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "go.opentelemetry.io/otel/metric/noop" "golang.org/x/net/dns/dnsmessage" ) @@ -124,7 +125,7 @@ func TestNewRelayRejects(t *testing.T) { if err := os.WriteFile(empty, nil, 0o644); err != nil { t.Fatal(err) } - if _, err := NewRelay(empty); err == nil { + if _, err := NewRelay(empty, noop.Meter{}); err == nil { t.Error("NewRelay succeeded with an empty resolv.conf") } if _, err := NewRelayForUpstreams(nil); err == nil { diff --git a/internal/atunnel/dns/resolvconf.go b/internal/atunnel/dns/resolvconf.go index 5b58c7a044..18c29cfe60 100644 --- a/internal/atunnel/dns/resolvconf.go +++ b/internal/atunnel/dns/resolvconf.go @@ -23,21 +23,25 @@ import ( "strings" ) -// resolvConfNameservers reads nameservers from resolv.conf as "host:53". -func resolvConfNameservers(path string) ([]string, error) { +// readResolvConf reads nameservers from resolv.conf as "host:53", and the +// search domains of its last search line. +func readResolvConf(path string) (nameservers, search []string, err error) { f, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) + return nil, nil, fmt.Errorf("dns: reading resolv.conf: %w", err) } defer f.Close() - var out []string scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if i := strings.IndexAny(line, "#;"); i >= 0 { line = strings.TrimSpace(line[:i]) } + if fields := strings.Fields(line); len(fields) > 0 && fields[0] == "search" { + search = fields[1:] + continue + } rest, ok := strings.CutPrefix(line, "nameserver") if !ok { continue @@ -46,13 +50,13 @@ func resolvConfNameservers(path string) ([]string, error) { if address == "" || net.ParseIP(address) == nil { continue } - out = append(out, net.JoinHostPort(address, "53")) + nameservers = append(nameservers, net.JoinHostPort(address, "53")) } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) + return nil, nil, fmt.Errorf("dns: reading resolv.conf: %w", err) } - if len(out) == 0 { - return nil, fmt.Errorf("dns: %s names no usable nameserver", path) + if len(nameservers) == 0 { + return nil, nil, fmt.Errorf("dns: %s names no usable nameserver", path) } - return out, nil + return nameservers, search, nil } diff --git a/internal/atunnel/dns/resolvconf_test.go b/internal/atunnel/dns/resolvconf_test.go index 400d94ce4c..01c090a315 100644 --- a/internal/atunnel/dns/resolvconf_test.go +++ b/internal/atunnel/dns/resolvconf_test.go @@ -22,17 +22,25 @@ import ( "github.com/google/go-cmp/cmp" ) -func TestResolvConfNameservers(t *testing.T) { +func TestReadResolvConf(t *testing.T) { for _, tc := range []struct { - name string - content string - want []string - wantErr bool + name string + content string + want []string + wantSearch []string + wantErr bool }{ { - name: "cluster resolv.conf", - content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", - want: []string{"10.96.0.10:53"}, + name: "cluster resolv.conf", + content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", + want: []string{"10.96.0.10:53"}, + wantSearch: []string{"ate-system.svc.cluster.local", "svc.cluster.local"}, + }, + { + name: "last search line wins", + content: "search first.example\nnameserver 10.96.0.10\nsearch cluster.local google.internal # node\n", + want: []string{"10.96.0.10:53"}, + wantSearch: []string{"cluster.local", "google.internal"}, }, { name: "several, in order", @@ -52,19 +60,22 @@ func TestResolvConfNameservers(t *testing.T) { if err := os.WriteFile(path, []byte(tc.content), 0o644); err != nil { t.Fatal(err) } - got, err := resolvConfNameservers(path) + got, gotSearch, err := readResolvConf(path) if tc.wantErr { if err == nil { - t.Fatalf("resolvConfNameservers() = %v, want an error", got) + t.Fatalf("readResolvConf() = %v, want an error", got) } return } if err != nil { - t.Fatalf("resolvConfNameservers: %v", err) + t.Fatalf("readResolvConf: %v", err) } if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("nameservers mismatch (-want +got):\n%s", diff) } + if diff := cmp.Diff(tc.wantSearch, gotSearch); diff != "" { + t.Errorf("search domains mismatch (-want +got):\n%s", diff) + } }) } } diff --git a/internal/atunnel/dns/server.go b/internal/atunnel/dns/server.go index 8fe8a5f080..8a5570710a 100644 --- a/internal/atunnel/dns/server.go +++ b/internal/atunnel/dns/server.go @@ -29,6 +29,8 @@ type serverConfig struct { // policy supplies the actor's egress policy, which each query is checked // against. Nil refuses every query. policy PolicySource + // instruments counts the policy decisions. Nil records nothing. + instruments *instruments } type netConn struct { @@ -88,7 +90,7 @@ func newServer( s.pendingRequests = newPendingRequests(s.limiter) // The check fetches with serveCtx, so that Stop cancels one in progress. - s.dns = newDNSHandler(s.pendingRequests, s.limiter, egressPolicyCheck(serveCtx, config.policy)) + s.dns = newDNSHandler(s.pendingRequests, s.limiter, egressPolicyCheck(serveCtx, config.policy), config.instruments) s.udp = newUDPHandler(netC.udp, netC.egressUDP, udpAddrs, s.dns, s.pendingRequests) s.tcp = newTCPHandler(config.upstreams, netC.tcpListener, netC.dialer, s.dns, s.limiter) diff --git a/internal/atunnel/dns/tcphandler_test.go b/internal/atunnel/dns/tcphandler_test.go index 006d363f6c..fc54b4d6d3 100644 --- a/internal/atunnel/dns/tcphandler_test.go +++ b/internal/atunnel/dns/tcphandler_test.go @@ -34,7 +34,7 @@ func startTestTCPHandler(t *testing.T, upstreams []string, configure func(*tcpHa } lim := newLimiter() - dnsH := newDNSHandler(nil, lim, nil) + dnsH := newDNSHandler(nil, lim, nil, nil) h := newTCPHandler(upstreams, lis, net.Dialer{Timeout: 5 * time.Second}, dnsH, lim) if configure != nil { configure(h, dnsH) diff --git a/internal/atunnel/dns/udphandler_test.go b/internal/atunnel/dns/udphandler_test.go index a6ab35fbe8..4fad99e4b3 100644 --- a/internal/atunnel/dns/udphandler_test.go +++ b/internal/atunnel/dns/udphandler_test.go @@ -50,7 +50,7 @@ func startTestUDPHandler(t *testing.T, upstreams []string, configure func(*udpHa lim := newLimiter() pending := newPendingRequests(lim) - dnsH := newDNSHandler(pending, lim, nil) + dnsH := newDNSHandler(pending, lim, nil, nil) h := newUDPHandler(ingress, egress, udpAddrs, dnsH, pending) if configure != nil { configure(h, dnsH) diff --git a/internal/atunnel/tunnel.go b/internal/atunnel/tunnel.go index d27d29fe7f..6904b83047 100644 --- a/internal/atunnel/tunnel.go +++ b/internal/atunnel/tunnel.go @@ -29,6 +29,8 @@ import ( "net/url" "time" + "go.opentelemetry.io/otel/metric" + "go.ate.dev/substrate/internal/ateletdial" "go.ate.dev/substrate/internal/atunnel/dns" "go.ate.dev/substrate/internal/atunnel/egress" @@ -112,8 +114,8 @@ type Tunnel struct { // Start builds the DNS relay and starts serving actor ingress, which proxies // to upstream inside each sandbox. Egress binds only in sandbox namespaces, so -// nothing listens for it here. -func Start(ctx context.Context, cfg Config, upstream string) (*Tunnel, error) { +// nothing listens for it here. The DNS relay reports its metrics on meter. +func Start(ctx context.Context, cfg Config, upstream string, meter metric.Meter) (*Tunnel, error) { if err := egresspolicy.ValidateDNSEgressPolicy(cfg.DNSEgressPolicy); err != nil { return nil, err } @@ -122,7 +124,7 @@ func Start(ctx context.Context, cfg Config, upstream string) (*Tunnel, error) { return nil, fmt.Errorf("while parsing atunnel upstream: %w", err) } - dnsRelay, err := dns.NewRelay(resolvConfPath) + dnsRelay, err := dns.NewRelay(resolvConfPath, meter) if err != nil { return nil, fmt.Errorf("configuring DNS relay: %w", err) } diff --git a/internal/atunnel/tunnel_test.go b/internal/atunnel/tunnel_test.go index 629739cf60..200d19ed06 100644 --- a/internal/atunnel/tunnel_test.go +++ b/internal/atunnel/tunnel_test.go @@ -38,6 +38,7 @@ import ( "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/internal/substratex509" + "go.opentelemetry.io/otel/metric/noop" ) func TestRegisterFlags(t *testing.T) { @@ -203,7 +204,7 @@ func startTunnel(t *testing.T, cfg Config) *Tunnel { t.Helper() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - tunnel, err := Start(ctx, cfg, "http://169.254.17.2:80") + tunnel, err := Start(ctx, cfg, "http://169.254.17.2:80", noop.Meter{}) if err != nil { t.Fatal(err) } @@ -284,7 +285,7 @@ func TestStartErrors(t *testing.T) { tc.edit(t, &cfg) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err := Start(ctx, cfg, "http://169.254.17.2:80") + _, err := Start(ctx, cfg, "http://169.254.17.2:80", noop.Meter{}) if (err != nil) != tc.wantErr { t.Fatalf("Start error = %v, wantErr = %v", err, tc.wantErr) } @@ -292,7 +293,7 @@ func TestStartErrors(t *testing.T) { } t.Run("bad upstream", func(t *testing.T) { - _, err := Start(context.Background(), testConfig(t), "http://[::1") + _, err := Start(context.Background(), testConfig(t), "http://[::1", noop.Meter{}) if err == nil { t.Fatalf("Start error = %v, want an error", err) } From 45e027fce3054c7cfabbb2d341a5fc7cc7b877cd Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Tue, 3 Nov 2026 17:05:28 -0800 Subject: [PATCH 2/3] remove the denied_search_domain metric Signed-off-by: Lior Lieberman --- docs/guides/observability.md | 2 +- docs/metrics/registry/metrics.yaml | 20 +++------ internal/ateattr/ateattr.go | 5 +-- internal/atunnel/dns/dnshandler.go | 2 +- internal/atunnel/dns/metrics.go | 37 ++++------------ internal/atunnel/dns/metrics_test.go | 57 +++++++++---------------- internal/atunnel/dns/relay.go | 4 +- internal/atunnel/dns/resolvconf.go | 22 ++++------ internal/atunnel/dns/resolvconf_test.go | 33 +++++--------- 9 files changed, 61 insertions(+), 121 deletions(-) diff --git a/docs/guides/observability.md b/docs/guides/observability.md index c248eb741e..0fc3dc7b30 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -305,7 +305,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.filecache.size` | atelet | up/down counter | bytes of disk a node-local file cache occupies, by `ate.filecache.name`, read on each collection; compare with `--sandbox-asset-cache-max-bytes` | | `ate.filecache.evictions` | atelet | counter | entries the eviction passes removed from a node-local file cache, by `ate.filecache.name`; a steady rate at the budget means the cache thrashes and actors pay for downloads again | | `ate.filecache.gc.passes` | atelet | counter | eviction passes over a node-local file cache, by `ate.filecache.name`, that had work to do, by outcome (`ate.filecache.gc.outcome`: `complete`, `shortfall`, `skipped`); a pass within budget is not counted | -| `ate.dns.queries` | ateworker-gvisor & ateworker-microvm | counter | actor DNS queries the sandbox DNS relay checked against the actor's egress policy, by `ate.dns.decision`: `allowed`, `denied` (answered `NXDOMAIN`), or `denied_search_domain` (a denied name under the worker pod's search domains, which resolvers try before most names) | +| `ate.dns.queries` | ateworker-gvisor & ateworker-microvm | counter | actor DNS queries the sandbox DNS relay checked against the actor's egress policy, by `ate.dns.decision` (`allowed` or `denied`); a denied query gets `NXDOMAIN`, and most lookups also cause denials for the worker pod's search domains, which resolvers try first | The table lists the OpenTelemetry instrument names. How a name appears in a query depends on the backend (Cloud Monitoring (GMP) / Kind collector). diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 6ed1e29e0b..3c21c7e1b3 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -657,15 +657,6 @@ groups: policy and holds no earlier copy. The relay answers NXDOMAIN. Thus the actor sees the same error as for a name that does not exist. - - id: denied_search_domain - stability: development - value: denied_search_domain - brief: > - The same as denied, for a name under one of the search domains - of the worker pod. The sandbox keeps those search domains and - ndots:5. Thus a stub resolver tries them before most names, and - most lookups cause these denials. A denied cluster name, such as - a service in another namespace, is also counted here. - id: registry.ate.stats type: attribute_group @@ -1643,10 +1634,13 @@ groups: stability: development brief: The number of actor DNS queries that the DNS relay of the sandbox checked against the egress policy, by decision. note: > - A denied rate above zero means that egress policies block external names - that actors look up. The actor sees only "bad address" or "could not - resolve host". The relay does not count a query that it drops or rejects - before the policy check, such as a malformed query. + The sandbox keeps the search domains of the worker pod and ndots:5. Thus + a stub resolver tries the search domains before most names, and most + lookups cause denials, even when a rule allows the name. Compare the + denied rate with its usual level. A rise means that egress policies block + names that actors look up. The actor sees only "bad address" or "could + not resolve host". The relay does not count a query that it drops or + rejects before the policy check, such as a malformed query. annotations: substrate: emitted_by: [ateworker-gvisor, ateworker-microvm] diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index e2f13da10e..e7161cc1a2 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -243,9 +243,8 @@ const ( // Values for DNSDecisionKey: whether the actor's egress policy let the sandbox // DNS relay forward a query. The relay answers a denied query with NXDOMAIN. const ( - DNSDecisionAllowed = "allowed" - DNSDecisionDenied = "denied" - DNSDecisionDeniedSearchDomain = "denied_search_domain" + DNSDecisionAllowed = "allowed" + DNSDecisionDenied = "denied" ) // Values for FileCacheNameKey. diff --git a/internal/atunnel/dns/dnshandler.go b/internal/atunnel/dns/dnshandler.go index c3b22e5f15..367b1bb919 100644 --- a/internal/atunnel/dns/dnshandler.go +++ b/internal/atunnel/dns/dnshandler.go @@ -125,7 +125,7 @@ func (h *dnsHandler) onRequestCommon(raw []byte) (dnsmessage.Header, dnsmessage. // error codes such as REFUSED. cq := canonicalizeQuestion(q) allowed := h.allow(cq) - h.instruments.recordQuery(cq, allowed) + h.instruments.recordQuery(allowed) if !allowed { return hdr, dnsmessage.Question{}, action{ kind: actionReply, diff --git a/internal/atunnel/dns/metrics.go b/internal/atunnel/dns/metrics.go index 9a2e06a052..b407609432 100644 --- a/internal/atunnel/dns/metrics.go +++ b/internal/atunnel/dns/metrics.go @@ -17,10 +17,8 @@ package dns import ( "context" "fmt" - "strings" "go.opentelemetry.io/otel/metric" - "golang.org/x/net/dns/dnsmessage" "go.ate.dev/substrate/internal/ateattr" ) @@ -31,15 +29,10 @@ const queriesMetric = "ate.dns.queries" // no-op, so call sites need no guard. type instruments struct { queries metric.Int64Counter - // searchSuffixes are the worker pod's search domains as ".domain.", in - // lowercase. - searchSuffixes []string } -// newInstruments creates the DNS relay's instruments against meter. search -// holds the search domains of the worker pod, which the sandbox's resolv.conf -// keeps. -func newInstruments(meter metric.Meter, search []string) (*instruments, error) { +// newInstruments creates the DNS relay's instruments against meter. +func newInstruments(meter metric.Meter) (*instruments, error) { queries, err := meter.Int64Counter( queriesMetric, metric.WithUnit("{query}"), @@ -48,31 +41,17 @@ func newInstruments(meter metric.Meter, search []string) (*instruments, error) { if err != nil { return nil, fmt.Errorf("create %s counter: %w", queriesMetric, err) } - i := &instruments{queries: queries} - for _, d := range search { - i.searchSuffixes = append(i.searchSuffixes, "."+strings.ToLower(strings.TrimSuffix(d, "."))+".") - } - return i, nil + return &instruments{queries: queries}, nil } -// recordQuery counts one query the egress policy allowed or denied. q is -// canonical, so its name is in lowercase. -func (i *instruments) recordQuery(q dnsmessage.Question, allowed bool) { +// recordQuery counts one query the egress policy allowed or denied. +func (i *instruments) recordQuery(allowed bool) { if i == nil { return } - decision := ateattr.DNSDecisionAllowed - if !allowed { - decision = ateattr.DNSDecisionDenied - // A stub resolver tries the search domains before most names (ndots:5 - // in Kubernetes), so most lookups cause these denials. - name := q.Name.String() - for _, suffix := range i.searchSuffixes { - if strings.HasSuffix(name, suffix) { - decision = ateattr.DNSDecisionDeniedSearchDomain - break - } - } + decision := ateattr.DNSDecisionDenied + if allowed { + decision = ateattr.DNSDecisionAllowed } i.queries.Add(context.Background(), 1, metric.WithAttributes(ateattr.DNSDecisionKey.String(decision))) } diff --git a/internal/atunnel/dns/metrics_test.go b/internal/atunnel/dns/metrics_test.go index 59c0647b03..1a0e33a733 100644 --- a/internal/atunnel/dns/metrics_test.go +++ b/internal/atunnel/dns/metrics_test.go @@ -18,7 +18,6 @@ import ( "context" "os" "path/filepath" - "strings" "testing" "github.com/google/go-cmp/cmp" @@ -69,47 +68,36 @@ func collectQueries(t *testing.T, reader *sdkmetric.ManualReader) map[string]int // policy and is not counted. func TestQueriesCountsPolicyDecisions(t *testing.T) { reader := sdkmetric.NewManualReader() - meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor") - inst, err := newInstruments(meter, []string{"ate-workers.svc.cluster.local", "SVC.Cluster.Local."}) + inst, err := newInstruments(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor")) if err != nil { t.Fatalf("newInstruments: %v", err) } lim := newLimiter() - allow := func(q dnsmessage.Question) bool { return !strings.HasPrefix(q.Name.String(), "blocked.") } + allow := func(q dnsmessage.Question) bool { return q.Name.String() != "blocked.example.com." } h := newDNSHandler(newPendingRequests(lim), lim, allow, inst) + upstreams := []string{"10.96.0.10:53"} - for i, name := range []string{ - "example.com.", - "example.com.svc.cluster.local.", - "blocked.example.com.", - "blocked.notsvc.cluster.local.", - "blocked.example.com.ate-workers.svc.cluster.local.", - "Blocked.Example.COM.svc.CLUSTER.local.", - } { - raw, _ := dnsQueryForName(t, uint16(i), name) - h.onTCPRequest(raw) - } - allowed, _ := dnsQueryForName(t, 100, "example.com.") - h.onUDPRequest(allowed, nil, []string{"10.96.0.10:53"}) - denied, _ := dnsQueryForName(t, 101, "blocked.example.com.") - h.onUDPRequest(denied, nil, []string{"10.96.0.10:53"}) - h.onTCPRequest(buildTestMessage(t, dnsmessage.Header{ID: 102}, nil)) + allowed, _ := dnsQueryForName(t, 1, "example.com.") + denied, _ := dnsQueryForName(t, 2, "blocked.example.com.") + h.onUDPRequest(allowed, nil, upstreams) + h.onTCPRequest(allowed) + h.onUDPRequest(denied, nil, upstreams) + h.onTCPRequest(denied) + h.onTCPRequest(buildTestMessage(t, dnsmessage.Header{ID: 3}, nil)) want := map[string]int64{ - ateattr.DNSDecisionAllowed: 3, - ateattr.DNSDecisionDenied: 3, - ateattr.DNSDecisionDeniedSearchDomain: 2, + ateattr.DNSDecisionAllowed: 2, + ateattr.DNSDecisionDenied: 2, } if diff := cmp.Diff(want, collectQueries(t, reader)); diff != "" { t.Errorf("ate.dns.queries mismatch (-want +got):\n%s", diff) } } -// TestNewRelayCountsQueries: a relay from NewRelay counts on its meter and -// takes the search domains from the resolv.conf it reads. +// TestNewRelayCountsQueries: a relay from NewRelay counts on its meter. func TestNewRelayCountsQueries(t *testing.T) { path := filepath.Join(t.TempDir(), "resolv.conf") - if err := os.WriteFile(path, []byte("search ate-workers.svc.cluster.local\nnameserver 127.0.0.1\n"), 0o644); err != nil { + if err := os.WriteFile(path, []byte("nameserver 127.0.0.1\n"), 0o644); err != nil { t.Fatal(err) } reader := sdkmetric.NewManualReader() @@ -117,22 +105,17 @@ func TestNewRelayCountsQueries(t *testing.T) { if err != nil { t.Fatalf("NewRelay: %v", err) } - // The actor has no policy, so the relay answers every query itself. + // The actor has no policy, so the relay answers the query itself. _, udpAddr, _ := serveLoopbackWithPolicy(t, relay, staticPolicy{}) conn := dialRelay(t, "udp", udpAddr) - for i, name := range []string{"example.com.", "example.com.ate-workers.svc.cluster.local."} { - query, q := dnsQueryForName(t, uint16(i), name) - if _, err := conn.Write(query); err != nil { - t.Fatal(err) - } - checkAnswerForQuestion(t, readWithin(t, conn), uint16(i), q, dnsmessage.RCodeNameError) + query, q := dnsQueryForName(t, 1, "example.com.") + if _, err := conn.Write(query); err != nil { + t.Fatal(err) } + checkAnswerForQuestion(t, readWithin(t, conn), 1, q, dnsmessage.RCodeNameError) - want := map[string]int64{ - ateattr.DNSDecisionDenied: 1, - ateattr.DNSDecisionDeniedSearchDomain: 1, - } + want := map[string]int64{ateattr.DNSDecisionDenied: 1} if diff := cmp.Diff(want, collectQueries(t, reader)); diff != "" { t.Errorf("ate.dns.queries mismatch (-want +got):\n%s", diff) } diff --git a/internal/atunnel/dns/relay.go b/internal/atunnel/dns/relay.go index 348d653935..86aa3adf65 100644 --- a/internal/atunnel/dns/relay.go +++ b/internal/atunnel/dns/relay.go @@ -48,7 +48,7 @@ type Relay struct { // NewRelay reads nameservers from resolvConfPath and forwards to each on port 53. // It reports its metrics on meter. func NewRelay(resolvConfPath string, meter metric.Meter) (*Relay, error) { - upstreams, search, err := readResolvConf(resolvConfPath) + upstreams, err := resolvConfNameservers(resolvConfPath) if err != nil { return nil, err } @@ -56,7 +56,7 @@ func NewRelay(resolvConfPath string, meter metric.Meter) (*Relay, error) { if err != nil { return nil, err } - if r.instruments, err = newInstruments(meter, search); err != nil { + if r.instruments, err = newInstruments(meter); err != nil { return nil, err } return r, nil diff --git a/internal/atunnel/dns/resolvconf.go b/internal/atunnel/dns/resolvconf.go index 18c29cfe60..5b58c7a044 100644 --- a/internal/atunnel/dns/resolvconf.go +++ b/internal/atunnel/dns/resolvconf.go @@ -23,25 +23,21 @@ import ( "strings" ) -// readResolvConf reads nameservers from resolv.conf as "host:53", and the -// search domains of its last search line. -func readResolvConf(path string) (nameservers, search []string, err error) { +// resolvConfNameservers reads nameservers from resolv.conf as "host:53". +func resolvConfNameservers(path string) ([]string, error) { f, err := os.Open(path) if err != nil { - return nil, nil, fmt.Errorf("dns: reading resolv.conf: %w", err) + return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) } defer f.Close() + var out []string scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if i := strings.IndexAny(line, "#;"); i >= 0 { line = strings.TrimSpace(line[:i]) } - if fields := strings.Fields(line); len(fields) > 0 && fields[0] == "search" { - search = fields[1:] - continue - } rest, ok := strings.CutPrefix(line, "nameserver") if !ok { continue @@ -50,13 +46,13 @@ func readResolvConf(path string) (nameservers, search []string, err error) { if address == "" || net.ParseIP(address) == nil { continue } - nameservers = append(nameservers, net.JoinHostPort(address, "53")) + out = append(out, net.JoinHostPort(address, "53")) } if err := scanner.Err(); err != nil { - return nil, nil, fmt.Errorf("dns: reading resolv.conf: %w", err) + return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) } - if len(nameservers) == 0 { - return nil, nil, fmt.Errorf("dns: %s names no usable nameserver", path) + if len(out) == 0 { + return nil, fmt.Errorf("dns: %s names no usable nameserver", path) } - return nameservers, search, nil + return out, nil } diff --git a/internal/atunnel/dns/resolvconf_test.go b/internal/atunnel/dns/resolvconf_test.go index 01c090a315..400d94ce4c 100644 --- a/internal/atunnel/dns/resolvconf_test.go +++ b/internal/atunnel/dns/resolvconf_test.go @@ -22,25 +22,17 @@ import ( "github.com/google/go-cmp/cmp" ) -func TestReadResolvConf(t *testing.T) { +func TestResolvConfNameservers(t *testing.T) { for _, tc := range []struct { - name string - content string - want []string - wantSearch []string - wantErr bool + name string + content string + want []string + wantErr bool }{ { - name: "cluster resolv.conf", - content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", - want: []string{"10.96.0.10:53"}, - wantSearch: []string{"ate-system.svc.cluster.local", "svc.cluster.local"}, - }, - { - name: "last search line wins", - content: "search first.example\nnameserver 10.96.0.10\nsearch cluster.local google.internal # node\n", - want: []string{"10.96.0.10:53"}, - wantSearch: []string{"cluster.local", "google.internal"}, + name: "cluster resolv.conf", + content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", + want: []string{"10.96.0.10:53"}, }, { name: "several, in order", @@ -60,22 +52,19 @@ func TestReadResolvConf(t *testing.T) { if err := os.WriteFile(path, []byte(tc.content), 0o644); err != nil { t.Fatal(err) } - got, gotSearch, err := readResolvConf(path) + got, err := resolvConfNameservers(path) if tc.wantErr { if err == nil { - t.Fatalf("readResolvConf() = %v, want an error", got) + t.Fatalf("resolvConfNameservers() = %v, want an error", got) } return } if err != nil { - t.Fatalf("readResolvConf: %v", err) + t.Fatalf("resolvConfNameservers: %v", err) } if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("nameservers mismatch (-want +got):\n%s", diff) } - if diff := cmp.Diff(tc.wantSearch, gotSearch); diff != "" { - t.Errorf("search domains mismatch (-want +got):\n%s", diff) - } }) } } From f9c8c4019453ab9ce7d8ee60bae6369aa532a418 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Fri, 13 Nov 2026 10:49:15 -0800 Subject: [PATCH 3/3] Rename the DNS relay's instruments to metrics Signed-off-by: Lior Lieberman --- internal/atunnel/dns/dnshandler.go | 20 ++++++++++---------- internal/atunnel/dns/metrics.go | 16 ++++++++-------- internal/atunnel/dns/metrics_test.go | 6 +++--- internal/atunnel/dns/readme.md | 2 +- internal/atunnel/dns/relay.go | 6 +++--- internal/atunnel/dns/server.go | 6 +++--- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/internal/atunnel/dns/dnshandler.go b/internal/atunnel/dns/dnshandler.go index 367b1bb919..46bc2ee2e6 100644 --- a/internal/atunnel/dns/dnshandler.go +++ b/internal/atunnel/dns/dnshandler.go @@ -58,21 +58,21 @@ type queryPolicy func(q dnsmessage.Question) bool // dnsHandler inspects DNS requests and responses and returns transport-agnostic // routing decisions. type dnsHandler struct { - pending *pendingRequests - limiter *limiter - allow queryPolicy - instruments *instruments + pending *pendingRequests + limiter *limiter + allow queryPolicy + metrics *metrics } -func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy, instruments *instruments) *dnsHandler { +func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy, metrics *metrics) *dnsHandler { if allow == nil { allow = func(dnsmessage.Question) bool { return true } } return &dnsHandler{ - pending: pending, - limiter: lim, - allow: allow, - instruments: instruments, + pending: pending, + limiter: lim, + allow: allow, + metrics: metrics, } } @@ -125,7 +125,7 @@ func (h *dnsHandler) onRequestCommon(raw []byte) (dnsmessage.Header, dnsmessage. // error codes such as REFUSED. cq := canonicalizeQuestion(q) allowed := h.allow(cq) - h.instruments.recordQuery(allowed) + h.metrics.recordQuery(allowed) if !allowed { return hdr, dnsmessage.Question{}, action{ kind: actionReply, diff --git a/internal/atunnel/dns/metrics.go b/internal/atunnel/dns/metrics.go index b407609432..00759ae9e7 100644 --- a/internal/atunnel/dns/metrics.go +++ b/internal/atunnel/dns/metrics.go @@ -25,14 +25,14 @@ import ( const queriesMetric = "ate.dns.queries" -// instruments holds the DNS relay's instruments. A nil *instruments is a valid +// metrics holds the DNS relay's metric instruments. A nil *metrics is a valid // no-op, so call sites need no guard. -type instruments struct { +type metrics struct { queries metric.Int64Counter } -// newInstruments creates the DNS relay's instruments against meter. -func newInstruments(meter metric.Meter) (*instruments, error) { +// newMetrics creates the DNS relay's metric instruments against meter. +func newMetrics(meter metric.Meter) (*metrics, error) { queries, err := meter.Int64Counter( queriesMetric, metric.WithUnit("{query}"), @@ -41,17 +41,17 @@ func newInstruments(meter metric.Meter) (*instruments, error) { if err != nil { return nil, fmt.Errorf("create %s counter: %w", queriesMetric, err) } - return &instruments{queries: queries}, nil + return &metrics{queries: queries}, nil } // recordQuery counts one query the egress policy allowed or denied. -func (i *instruments) recordQuery(allowed bool) { - if i == nil { +func (m *metrics) recordQuery(allowed bool) { + if m == nil { return } decision := ateattr.DNSDecisionDenied if allowed { decision = ateattr.DNSDecisionAllowed } - i.queries.Add(context.Background(), 1, metric.WithAttributes(ateattr.DNSDecisionKey.String(decision))) + m.queries.Add(context.Background(), 1, metric.WithAttributes(ateattr.DNSDecisionKey.String(decision))) } diff --git a/internal/atunnel/dns/metrics_test.go b/internal/atunnel/dns/metrics_test.go index 1a0e33a733..cb0bfda52d 100644 --- a/internal/atunnel/dns/metrics_test.go +++ b/internal/atunnel/dns/metrics_test.go @@ -68,13 +68,13 @@ func collectQueries(t *testing.T, reader *sdkmetric.ManualReader) map[string]int // policy and is not counted. func TestQueriesCountsPolicyDecisions(t *testing.T) { reader := sdkmetric.NewManualReader() - inst, err := newInstruments(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor")) + m, err := newMetrics(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("ateworker-gvisor")) if err != nil { - t.Fatalf("newInstruments: %v", err) + t.Fatalf("newMetrics: %v", err) } lim := newLimiter() allow := func(q dnsmessage.Question) bool { return q.Name.String() != "blocked.example.com." } - h := newDNSHandler(newPendingRequests(lim), lim, allow, inst) + h := newDNSHandler(newPendingRequests(lim), lim, allow, m) upstreams := []string{"10.96.0.10:53"} allowed, _ := dnsQueryForName(t, 1, "example.com.") diff --git a/internal/atunnel/dns/readme.md b/internal/atunnel/dns/readme.md index ef9e0befd8..14225bbbc1 100644 --- a/internal/atunnel/dns/readme.md +++ b/internal/atunnel/dns/readme.md @@ -73,4 +73,4 @@ UDP and TCP call up to the DNS protocol layer (`golang.org/x/net/dns/dnsmessage` - `evaluator` (`evaluator.go`) compiles the hostname and wildcard patterns from an `EgressPolicy`'s `http`, `https`, and `tls_passthrough` rules and evaluates normalized `QNAME`s via `allowsHostname(qname)`. - `egressPolicyCheck` adapts the source to the `dnsHandler`'s `queryPolicy` hook, caching the compiled `*evaluator` for the current `*ateletpb.EgressPolicy` pointer and consulting it for every query. Fetching uses the `Server`'s context, so `Stop()` cancels a fetch in progress. - `allowQuery` evaluates a single `dnsmessage.Question` against a policy using `evaluator`. -- `instruments` (`metrics.go`) counts each decision on `ate.dns.queries`, by `ate.dns.decision`. +- `metrics` (`metrics.go`) counts each decision on `ate.dns.queries`, by `ate.dns.decision`. diff --git a/internal/atunnel/dns/relay.go b/internal/atunnel/dns/relay.go index 86aa3adf65..b716364608 100644 --- a/internal/atunnel/dns/relay.go +++ b/internal/atunnel/dns/relay.go @@ -42,7 +42,7 @@ type Relay struct { // dialer reaches upstream resolvers from the worker namespace. dialer *net.Dialer - instruments *instruments + metrics *metrics } // NewRelay reads nameservers from resolvConfPath and forwards to each on port 53. @@ -56,7 +56,7 @@ func NewRelay(resolvConfPath string, meter metric.Meter) (*Relay, error) { if err != nil { return nil, err } - if r.instruments, err = newInstruments(meter); err != nil { + if r.metrics, err = newMetrics(meter); err != nil { return nil, err } return r, nil @@ -110,5 +110,5 @@ func NewRelayForUpstreams(upstreams []string) (*Relay, error) { // serveOn serves DNS on netC's sockets, which the caller has already bound, // and dials upstreams with netC's dialer. func (r *Relay) serveOn(ctx context.Context, netC *netConn, policy PolicySource) (*Server, error) { - return newServer(ctx, &serverConfig{upstreams: r.upstreams, policy: policy, instruments: r.instruments}, netC) + return newServer(ctx, &serverConfig{upstreams: r.upstreams, policy: policy, metrics: r.metrics}, netC) } diff --git a/internal/atunnel/dns/server.go b/internal/atunnel/dns/server.go index 8a5570710a..4cc7065ff3 100644 --- a/internal/atunnel/dns/server.go +++ b/internal/atunnel/dns/server.go @@ -29,8 +29,8 @@ type serverConfig struct { // policy supplies the actor's egress policy, which each query is checked // against. Nil refuses every query. policy PolicySource - // instruments counts the policy decisions. Nil records nothing. - instruments *instruments + // metrics counts the policy decisions. Nil records nothing. + metrics *metrics } type netConn struct { @@ -90,7 +90,7 @@ func newServer( s.pendingRequests = newPendingRequests(s.limiter) // The check fetches with serveCtx, so that Stop cancels one in progress. - s.dns = newDNSHandler(s.pendingRequests, s.limiter, egressPolicyCheck(serveCtx, config.policy), config.instruments) + s.dns = newDNSHandler(s.pendingRequests, s.limiter, egressPolicyCheck(serveCtx, config.policy), config.metrics) s.udp = newUDPHandler(netC.udp, netC.egressUDP, udpAddrs, s.dns, s.pendingRequests) s.tcp = newTCPHandler(config.upstreams, netC.tcpListener, netC.dialer, s.dns, s.limiter)