ate-setup: settings for the router's route timeout, parked budget and ext_proc breaker

cmd/ate-setup/commands.md

@@ -20,6 +20,9 @@ every setting with its environment variable.
 |---|---|
 | `--kind` | Kind overlays, the local registry, and host-architecture image builds. `ATE_INSTALL_KIND=true` when the flag is absent |
 | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) |
+| `--atenet-route-timeout DURATION` | The router's `--route-timeout` (default `5m`). Envoy dataplane only. `ATE_ATENET_ROUTE_TIMEOUT` when the flag is absent |
+| `--atenet-parked-request-budget DURATION` | The router's `--parked-request-budget` (default `5s`); the router pod's `terminationGracePeriodSeconds` grows with it. Envoy dataplane only. `ATE_ATENET_PARKED_REQUEST_BUDGET` when the flag is absent |
+| `--atenet-extproc-max-requests N` | The router's `--extproc-max-requests`, the circuit breaker on Envoy's ext_proc cluster (default: twice `--parked-request-max`, at least `1024`). Envoy dataplane only. `ATE_ATENET_EXTPROC_MAX_REQUESTS` when the flag is absent |
 | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). It also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed |
 | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer |
 | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent |

cmd/ate-setup/internal/config/config.go

@@ -167,6 +167,13 @@ type Config struct {
 
 	// Router selects the atenet router dataplane.
 	Router string
+	// RouterRouteTimeout and RouterParkedRequestBudget are the router's
+	// --route-timeout and --parked-request-budget; zero keeps the router default.
+	RouterRouteTimeout        time.Duration
+	RouterParkedRequestBudget time.Duration
+	// RouterExtProcMaxRequests is the router's --extproc-max-requests; zero
+	// keeps the router's derivation from --parked-request-max.
+	RouterExtProcMaxRequests int
 	// The read/write and owner connections can use different login identities.
 	// With one configured connection, both pools use it. Both empty selects
 	// bundled PostgreSQL.

cmd/ate-setup/internal/config/config_test.go

@@ -60,6 +60,9 @@ func loadEnv(t *testing.T) {
 		"ATE_API_POSTGRES_SCHEMA",
 		"ATE_API_POSTGRES_SERVER_CA_FILE",
 		"ATE_ATENET_DATAPLANE",
+		"ATE_ATENET_EXTPROC_MAX_REQUESTS",
+		"ATE_ATENET_PARKED_REQUEST_BUDGET",
+		"ATE_ATENET_ROUTE_TIMEOUT",
 		"ATE_CREDENTIAL_PROVIDER",
 		"ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS",
 		"ATE_IMAGE_REPO",

cmd/ate-setup/internal/config/load.go

@@ -238,11 +238,14 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro
 			Repo: strings.TrimSuffix(r.String("images.repo"), "/"),
 			Tag:  r.String("images.tag"),
 		},
-		Router:               r.String("atenet.dataplane"),
-		PostgresSchema:       r.String("ateapi.postgres.schema"),
-		PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"),
-		PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"),
-		PostgresStorageClass: r.String("ateapi.postgres.storageClass"),
+		Router:                    r.String("atenet.dataplane"),
+		RouterRouteTimeout:        r.Duration("atenet.router.routeTimeout"),
+		RouterParkedRequestBudget: r.Duration("atenet.router.parkedRequestBudget"),
+		RouterExtProcMaxRequests:  r.Int("atenet.router.extprocMaxRequests"),
+		PostgresSchema:            r.String("ateapi.postgres.schema"),
+		PostgresPoolMaxConns:      r.String("ateapi.postgres.poolMaxConns"),
+		PostgresServerCAFile:      r.String("ateapi.postgres.serverCAFile"),
+		PostgresStorageClass:      r.String("ateapi.postgres.storageClass"),
 
 		PostgresReadWriteConnectionString: readWriteConnectionString,
 		PostgresOwnerConnectionString:     ownerConnectionString,
@@ -394,6 +397,27 @@ func validateResolved(cfg *Config, r *Resolved) error {
 		return &InvalidError{Value: dataplane, Want: RouterEnvoy + " or " + RouterAgentgateway}
 	}
 
+	// Only the envoy dataplane runs the atenet-router container the flags go on.
+	for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget", "atenet.router.extprocMaxRequests"} {
+		v, _ := r.Value(key)
+		if v.Raw == "" {
+			continue
+		}
+		switch v.Setting.Kind {
+		case KindDuration:
+			if r.Duration(key) <= 0 {
+				return &InvalidError{Value: v, Want: "a positive duration"}
+			}
+		case KindInt:
+			if r.Int(key) <= 0 {
+				return &InvalidError{Value: v, Want: "a positive integer"}
+			}
+		}
+		if cfg.Router != RouterEnvoy {
+			return &ConflictError{A: v, B: dataplane, Why: "the router flags need the envoy dataplane"}
+		}
+	}
+
 	if ipType, _ := r.Value("ateapi.postgres.cloudsql.ipType"); ipType.Raw != "" {
 		switch ipType.Raw {
 		case CloudSQLIPTypePrivate, CloudSQLIPTypePublic, CloudSQLIPTypePSC:

cmd/ate-setup/internal/config/load_test.go

@@ -20,6 +20,7 @@ import (
 	"runtime"
 	"strings"
 	"testing"
+	"time"
 )
 
 // Under --kind the ambient environment is ignored, but a value the caller named
@@ -222,3 +223,76 @@ func errPath(err error) string {
 	}
 	return rest
 }
+
+func TestLoadFlagsRouterSettings(t *testing.T) {
+	loadEnv(t)
+	t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "30m")
+	t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m")
+	t.Setenv("ATE_ATENET_EXTPROC_MAX_REQUESTS", "4096")
+
+	fs, err := optionsFlagSet(Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := LoadFlags(fs, LoadOptions{})
+	if err != nil {
+		t.Fatalf("LoadFlags() error = %v", err)
+	}
+	if cfg.RouterRouteTimeout != 30*time.Minute {
+		t.Errorf("RouterRouteTimeout = %s, want 30m", cfg.RouterRouteTimeout)
+	}
+	if cfg.RouterParkedRequestBudget != 2*time.Minute {
+		t.Errorf("RouterParkedRequestBudget = %s, want 2m", cfg.RouterParkedRequestBudget)
+	}
+	if cfg.RouterExtProcMaxRequests != 4096 {
+		t.Errorf("RouterExtProcMaxRequests = %d, want 4096", cfg.RouterExtProcMaxRequests)
+	}
+}
+
+func TestLoadFlagsRouterSettingsNeedEnvoy(t *testing.T) {
+	for env, value := range map[string]string{
+		"ATE_ATENET_PARKED_REQUEST_BUDGET": "2m",
+		"ATE_ATENET_EXTPROC_MAX_REQUESTS":  "4096",
+	} {
+		t.Run(env, func(t *testing.T) {
+			loadEnv(t)
+			t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway)
+			t.Setenv(env, value)
+
+			fs, err := optionsFlagSet(Options{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			_, err = LoadFlags(fs, LoadOptions{})
+			if err == nil {
+				t.Fatal("LoadFlags() = nil error, want a dataplane conflict")
+			}
+			for _, want := range []string{env, "ATE_ATENET_DATAPLANE"} {
+				if !strings.Contains(err.Error(), want) {
+					t.Errorf("error %q does not name %s", err, want)
+				}
+			}
+		})
+	}
+}
+
+func TestLoadFlagsRouterSettingsMustBePositive(t *testing.T) {
+	for _, tc := range []struct{ env, value, want string }{
+		{"ATE_ATENET_ROUTE_TIMEOUT", "0s", "positive duration"},
+		{"ATE_ATENET_EXTPROC_MAX_REQUESTS", "0", "positive integer"},
+		{"ATE_ATENET_EXTPROC_MAX_REQUESTS", "-1", "positive integer"},
+	} {
+		t.Run(tc.env+"="+tc.value, func(t *testing.T) {
+			loadEnv(t)
+			t.Setenv(tc.env, tc.value)
+
+			fs, err := optionsFlagSet(Options{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Errorf("LoadFlags() error = %v, want a %s error", err, tc.want)
+			}
+		})
+	}
+}

cmd/ate-setup/internal/config/setting.go

@@ -163,6 +163,22 @@ var Registry = []Setting{
 		Default: RouterEnvoy,
 		Usage:   "Atenet ingress and egress dataplane: envoy or agentgateway",
 	},
+	{
+		// Set on the envoy dataplane's atenet-router container; see steps.patchAtenetRouterSettings.
+		Key: "atenet.router.routeTimeout", Env: "ATE_ATENET_ROUTE_TIMEOUT", Flag: "atenet-route-timeout",
+		Kind:  KindDuration,
+		Usage: "The router's --route-timeout; unset keeps the router default (envoy dataplane only)",
+	},
+	{
+		Key: "atenet.router.parkedRequestBudget", Env: "ATE_ATENET_PARKED_REQUEST_BUDGET", Flag: "atenet-parked-request-budget",
+		Kind:  KindDuration,
+		Usage: "The router's --parked-request-budget; unset keeps the router default; the router pod's grace period grows with it (envoy dataplane only)",
+	},
+	{
+		Key: "atenet.router.extprocMaxRequests", Env: "ATE_ATENET_EXTPROC_MAX_REQUESTS", Flag: "atenet-extproc-max-requests",
+		Kind:  KindInt,
+		Usage: "The router's --extproc-max-requests, the circuit breaker on Envoy's ext_proc cluster; unset keeps the router's derivation (envoy dataplane only)",
+	},
 	{
 		Key: "atenet.egress.additionalExtprocService", Env: "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE",
 		Flag: "experimental-additional-egress-extproc-service", Kind: KindString,

cmd/ate-setup/internal/steps/overlay.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -163,6 +164,10 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
+		manifest, err = e.patchAtenetRouterSettings(manifest)
+		if err != nil {
+			return nil, err
+		}
 		imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane)
 		if err != nil {
 			return nil, err
@@ -194,6 +199,10 @@ func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	manifest, err = e.patchAtenetRouterSettings(manifest)
+	if err != nil {
+		return nil, err
+	}
 	imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane)
 	if err != nil {
 		return nil, err
@@ -307,11 +316,64 @@ func (e *Env) patchAtenetRouterAllowedSPIFFEIDs(raw []byte) ([]byte, error) {
 		"--ingress-allowed-spiffe-ids="+strings.Join(e.Cfg.IngressAllowedSPIFFEIDs, ","))
 }
 
-// appendContainerArgs uses a kustomize JSON6902 patch to append args to the
-// first container of the named Deployment in ate-system. The patch first
-// tests that the first container is the one named, so a reordered manifest
-// fails the render instead of patching the wrong container.
+// The router manifest's terminationGracePeriodSeconds, and the
+// --parked-request-budget it is sized for, the router default.
+const (
+	routerManifestGracePeriodSeconds  = 60
+	routerManifestParkedRequestBudget = 5 * time.Second
+)
+
+// patchAtenetRouterSettings appends the flags the atenet.router.* settings set
+// to the atenet-router container's args. A parked request budget above the
+// manifest's grows terminationGracePeriodSeconds by the difference, since the
+// router's drain deadline follows the budget. Unset settings leave the
+// manifest alone.
+func (e *Env) patchAtenetRouterSettings(raw []byte) ([]byte, error) {
+	var args []string
+	if d := e.Cfg.RouterRouteTimeout; d > 0 {
+		args = append(args, "--route-timeout="+d.String())
+	}
+	if d := e.Cfg.RouterParkedRequestBudget; d > 0 {
+		args = append(args, "--parked-request-budget="+d.String())
+	}
+	if n := e.Cfg.RouterExtProcMaxRequests; n > 0 {
+		args = append(args, "--extproc-max-requests="+strconv.Itoa(n))
+	}
+	if len(args) == 0 {
+		return raw, nil
+	}
+	raw, err := appendContainerArgs(raw, "atenet-router", "atenet-router", args...)
+	if err != nil {
+		return nil, err
+	}
+	if extra := e.Cfg.RouterParkedRequestBudget - routerManifestParkedRequestBudget; extra > 0 {
+		grace := routerManifestGracePeriodSeconds + int64(math.Ceil(extra.Seconds()))
+		raw, err = patchDeployment(raw, "atenet-router", fmt.Sprintf(`- op: replace
+  path: /spec/template/spec/terminationGracePeriodSeconds
+  value: %d`, grace))
+	}
+	return raw, err
+}
+
+// appendContainerArgs appends args to the first container of the named
+// Deployment in ate-system. The patch first tests that the first container is
+// the one named, so a reordered manifest fails the render instead of patching
+// the wrong container.
 func appendContainerArgs(raw []byte, deployment, container string, args ...string) ([]byte, error) {
+	ops := []string{fmt.Sprintf(`- op: test
+  path: /spec/template/spec/containers/0/name
+  value: %q`, container)}
+	for _, arg := range args {
+		ops = append(ops, fmt.Sprintf(`- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: %q`, arg))
+	}
+	return patchDeployment(raw, deployment, ops...)
+}
+
+// patchDeployment applies ops, the entries of a JSON6902 patch in YAML, to the
+// named Deployment in ate-system through a scratch kustomization.
+func patchDeployment(raw []byte, deployment string, ops ...string) ([]byte, error) {
 	dir, err := os.MkdirTemp("", "ate-setup-kustomize-")
 	if err != nil {
 		return nil, fmt.Errorf("while creating a scratch kustomization: %w", err)
@@ -334,15 +396,11 @@ patches:
       name: %s
       namespace: ate-system
     patch: |-
-      - op: test
-        path: /spec/template/spec/containers/0/name
-        value: %q
-`, deployment, container)
-	for _, arg := range args {
-		fmt.Fprintf(&kustomization, `      - op: add
-        path: /spec/template/spec/containers/0/args/-
-        value: %q
-`, arg)
+`, deployment)
+	for _, op := range ops {
+		for line := range strings.SplitSeq(op, "\n") {
+			kustomization.WriteString("      " + line + "\n")
+		}
 	}
 	if err := os.WriteFile(filepath.Join(dir, "kustomization.yaml"), []byte(kustomization.String()), 0o600); err != nil {
 		return nil, fmt.Errorf("while writing a scratch kustomization: %w", err)

cmd/ate-setup/internal/steps/overlay_test.go

@@ -970,6 +970,84 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) {
 	}
 }
 
+// Both paths that deliver the Envoy router pass it the atenet.router.*
+// settings, and a parked request budget above the manifest's grows the pod's
+// grace period by the difference. Unset, the manifest comes out as checked in,
+// so it must not set the flags itself; that case also pins
+// routerManifestGracePeriodSeconds to the manifest.
+func TestRenderRouterSettings(t *testing.T) {
+	src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"}
+	resolver := images.NewPrebuilt(src, func(context.Context, string) (string, error) {
+		return "sha256:" + strings.Repeat("2", 64), nil
+	})
+	flags := []string{"--route-timeout=", "--parked-request-budget=", "--extproc-max-requests="}
+	for _, tc := range []struct {
+		name      string
+		cfg       config.Config
+		wantArgs  []string
+		wantGrace int64
+	}{
+		{name: "unset", wantGrace: routerManifestGracePeriodSeconds},
+		{
+			name:      "all set",
+			cfg:       config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute, RouterExtProcMaxRequests: 4096},
+			wantArgs:  []string{"--route-timeout=30m0s", "--parked-request-budget=2m0s", "--extproc-max-requests=4096"},
+			wantGrace: routerManifestGracePeriodSeconds + 115,
+		},
+		{
+			name:      "budget at the router default",
+			cfg:       config.Config{RouterParkedRequestBudget: 5 * time.Second},
+			wantArgs:  []string{"--parked-request-budget=5s"},
+			wantGrace: routerManifestGracePeriodSeconds,
+		},
+		{
+			name:      "breaker alone",
+			cfg:       config.Config{RouterExtProcMaxRequests: 4096},
+			wantArgs:  []string{"--extproc-max-requests=4096"},
+			wantGrace: routerManifestGracePeriodSeconds,
+		},
+	} {
+		for _, path := range []struct {
+			name   string
+			kind   bool
+			render func(e *Env) ([]byte, error)
+		}{
+			{name: "base bundle", render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }},
+			{name: "kind bundle", kind: true, render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }},
+			{name: "router file", render: func(e *Env) ([]byte, error) { return e.renderAtenetRouterManifest(t.Context()) }},
+		} {
+			t.Run(tc.name+"/"+path.name, func(t *testing.T) {
+				cfg := tc.cfg
+				cfg.Root = repoRoot(t)
+				cfg.Router = config.RouterEnvoy
+				cfg.Kind = path.kind
+				cfg.Images = src
+				rendered, err := path.render(&Env{Cfg: &cfg, resolver: resolver})
+				if err != nil {
+					t.Fatalf("render: %v", err)
+				}
+				var got []string
+				for _, arg := range routerArgs(t, rendered) {
+					if slices.ContainsFunc(flags, func(flag string) bool { return strings.HasPrefix(arg, flag) }) {
+						got = append(got, arg)
+					}
+				}
+				if !slices.Equal(got, tc.wantArgs) {
+					t.Errorf("atenet-router settings args = %q, want %q", got, tc.wantArgs)
+				}
+				_, deployments := dataPlaneWorkloads(t, rendered)
+				grace := deployments["atenet-router"].Spec.Template.Spec.TerminationGracePeriodSeconds
+				if grace == nil {
+					t.Fatal("the atenet-router pod spec has no terminationGracePeriodSeconds")
+				}
+				if *grace != tc.wantGrace {
+					t.Errorf("atenet-router terminationGracePeriodSeconds = %d, want %d", *grace, tc.wantGrace)
+				}
+			})
+		}
+	}
+}
+
 func TestRenderAgentgatewayStaticMTLS(t *testing.T) {
 	const (
 		idA = "spiffe://cluster.local/ns/a/sa/a"