From 5142e36725c9883997a2484dddae6fabfe491013 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sat, 10 Oct 2026 20:31:10 -0400 Subject: [PATCH 1/2] ate-setup: settings for the router's route timeout and parked budget --atenet-route-timeout and --atenet-parked-request-budget set the two router flags at install time; unset keeps the router defaults. A larger budget grows terminationGracePeriodSeconds by the same amount so parked requests finish during a rollout. Envoy dataplane only. The render functions patch the flags onto the atenet-router container beside --ingress-allowed-spiffe-ids, so both deploy paths carry them. A JSON6902 op sets the grace period. Signed-off-by: Davanum Srinivas --- cmd/ate-setup/commands.md | 2 + cmd/ate-setup/internal/config/config.go | 4 + cmd/ate-setup/internal/config/config_test.go | 2 + cmd/ate-setup/internal/config/load.go | 26 +++++-- cmd/ate-setup/internal/config/load_test.go | 55 +++++++++++++ cmd/ate-setup/internal/config/setting.go | 11 +++ cmd/ate-setup/internal/steps/overlay.go | 81 ++++++++++++++++---- cmd/ate-setup/internal/steps/overlay_test.go | 72 +++++++++++++++++ docs/concepts/request-parking.md | 11 +++ docs/examples/substrate-install.yaml | 4 + docs/guides/operator-install.md | 2 + manifests/ate-install/atenet-router.yaml | 2 + 12 files changed, 254 insertions(+), 18 deletions(-) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 0b91ed1390..d24826f26e 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -20,6 +20,8 @@ every setting with its environment variable. |---|---| | `--kind` | Kind overlays, the local registry, and host-architecture image builds. `ATE_INSTALL_KIND=true` when the flag is absent | | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | +| `--atenet-route-timeout DURATION` | The router's `--route-timeout` (default `5m`). Envoy dataplane only. `ATE_ATENET_ROUTE_TIMEOUT` when the flag is absent | +| `--atenet-parked-request-budget DURATION` | The router's `--parked-request-budget` (default `5s`); the router pod's `terminationGracePeriodSeconds` grows with it. Envoy dataplane only. `ATE_ATENET_PARKED_REQUEST_BUDGET` when the flag is absent | | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). It also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 30bcd18aad..1e16efbad5 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -167,6 +167,10 @@ type Config struct { // Router selects the atenet router dataplane. Router string + // RouterRouteTimeout and RouterParkedRequestBudget are the router's + // --route-timeout and --parked-request-budget; zero keeps the router default. + RouterRouteTimeout time.Duration + RouterParkedRequestBudget time.Duration // The read/write and owner connections can use different login identities. // With one configured connection, both pools use it. Both empty selects // bundled PostgreSQL. diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 18e95e41be..2e81d022ad 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -60,6 +60,8 @@ func loadEnv(t *testing.T) { "ATE_API_POSTGRES_SCHEMA", "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", + "ATE_ATENET_PARKED_REQUEST_BUDGET", + "ATE_ATENET_ROUTE_TIMEOUT", "ATE_CREDENTIAL_PROVIDER", "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", "ATE_IMAGE_REPO", diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index c9410fab3a..610932e5e4 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -238,11 +238,13 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro Repo: strings.TrimSuffix(r.String("images.repo"), "/"), Tag: r.String("images.tag"), }, - Router: r.String("atenet.dataplane"), - PostgresSchema: r.String("ateapi.postgres.schema"), - PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), - PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), - PostgresStorageClass: r.String("ateapi.postgres.storageClass"), + Router: r.String("atenet.dataplane"), + RouterRouteTimeout: r.Duration("atenet.router.routeTimeout"), + RouterParkedRequestBudget: r.Duration("atenet.router.parkedRequestBudget"), + PostgresSchema: r.String("ateapi.postgres.schema"), + PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), + PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), + PostgresStorageClass: r.String("ateapi.postgres.storageClass"), PostgresReadWriteConnectionString: readWriteConnectionString, PostgresOwnerConnectionString: ownerConnectionString, @@ -394,6 +396,20 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: dataplane, Want: RouterEnvoy + " or " + RouterAgentgateway} } + // Only the envoy dataplane runs the atenet-router container the flags go on. + for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget"} { + v, _ := r.Value(key) + if v.Raw == "" { + continue + } + if r.Duration(key) <= 0 { + return &InvalidError{Value: v, Want: "a positive duration"} + } + if cfg.Router != RouterEnvoy { + return &ConflictError{A: v, B: dataplane, Why: "the router flags need the envoy dataplane"} + } + } + if ipType, _ := r.Value("ateapi.postgres.cloudsql.ipType"); ipType.Raw != "" { switch ipType.Raw { case CloudSQLIPTypePrivate, CloudSQLIPTypePublic, CloudSQLIPTypePSC: diff --git a/cmd/ate-setup/internal/config/load_test.go b/cmd/ate-setup/internal/config/load_test.go index f9a620985b..2e2c2e5f4b 100644 --- a/cmd/ate-setup/internal/config/load_test.go +++ b/cmd/ate-setup/internal/config/load_test.go @@ -20,6 +20,7 @@ import ( "runtime" "strings" "testing" + "time" ) // Under --kind the ambient environment is ignored, but a value the caller named @@ -222,3 +223,57 @@ func errPath(err error) string { } return rest } + +func TestLoadFlagsRouterSettings(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "30m") + t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + cfg, err := LoadFlags(fs, LoadOptions{}) + if err != nil { + t.Fatalf("LoadFlags() error = %v", err) + } + if cfg.RouterRouteTimeout != 30*time.Minute { + t.Errorf("RouterRouteTimeout = %s, want 30m", cfg.RouterRouteTimeout) + } + if cfg.RouterParkedRequestBudget != 2*time.Minute { + t.Errorf("RouterParkedRequestBudget = %s, want 2m", cfg.RouterParkedRequestBudget) + } +} + +func TestLoadFlagsRouterSettingsNeedEnvoy(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway) + t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + _, err = LoadFlags(fs, LoadOptions{}) + if err == nil { + t.Fatal("LoadFlags() = nil error, want a dataplane conflict") + } + for _, want := range []string{"ATE_ATENET_PARKED_REQUEST_BUDGET", "ATE_ATENET_DATAPLANE"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %s", err, want) + } + } +} + +func TestLoadFlagsRouterSettingsMustBePositive(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "0s") + + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), "positive duration") { + t.Errorf("LoadFlags() error = %v, want a positive-duration error", err) + } +} diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index cb3dfd15c8..0c76802b65 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -163,6 +163,17 @@ var Registry = []Setting{ Default: RouterEnvoy, Usage: "Atenet ingress and egress dataplane: envoy or agentgateway", }, + { + // Set on the envoy dataplane's atenet-router container; see steps.patchAtenetRouterSettings. + Key: "atenet.router.routeTimeout", Env: "ATE_ATENET_ROUTE_TIMEOUT", Flag: "atenet-route-timeout", + Kind: KindDuration, + Usage: "The router's --route-timeout; unset keeps the router default (envoy dataplane only)", + }, + { + Key: "atenet.router.parkedRequestBudget", Env: "ATE_ATENET_PARKED_REQUEST_BUDGET", Flag: "atenet-parked-request-budget", + Kind: KindDuration, + Usage: "The router's --parked-request-budget; unset keeps the router default; the router pod's grace period grows with it (envoy dataplane only)", + }, { Key: "atenet.egress.additionalExtprocService", Env: "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", Flag: "experimental-additional-egress-extproc-service", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 2fbaf8ca66..0ea1134d09 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "math" "os" "path/filepath" "strconv" @@ -163,6 +164,10 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { if err != nil { return nil, err } + manifest, err = e.patchAtenetRouterSettings(manifest) + if err != nil { + return nil, err + } imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err @@ -194,6 +199,10 @@ func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { if err != nil { return nil, err } + manifest, err = e.patchAtenetRouterSettings(manifest) + if err != nil { + return nil, err + } imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err @@ -307,11 +316,61 @@ func (e *Env) patchAtenetRouterAllowedSPIFFEIDs(raw []byte) ([]byte, error) { "--ingress-allowed-spiffe-ids="+strings.Join(e.Cfg.IngressAllowedSPIFFEIDs, ",")) } -// appendContainerArgs uses a kustomize JSON6902 patch to append args to the -// first container of the named Deployment in ate-system. The patch first -// tests that the first container is the one named, so a reordered manifest -// fails the render instead of patching the wrong container. +// The router manifest's terminationGracePeriodSeconds, and the +// --parked-request-budget it is sized for, the router default. +const ( + routerManifestGracePeriodSeconds = 60 + routerManifestParkedRequestBudget = 5 * time.Second +) + +// patchAtenetRouterSettings appends the flags the atenet.router.* settings set +// to the atenet-router container's args. A parked request budget above the +// manifest's grows terminationGracePeriodSeconds by the difference, since the +// router's drain deadline follows the budget. Unset settings leave the +// manifest alone. +func (e *Env) patchAtenetRouterSettings(raw []byte) ([]byte, error) { + var args []string + if d := e.Cfg.RouterRouteTimeout; d > 0 { + args = append(args, "--route-timeout="+d.String()) + } + if d := e.Cfg.RouterParkedRequestBudget; d > 0 { + args = append(args, "--parked-request-budget="+d.String()) + } + if len(args) == 0 { + return raw, nil + } + raw, err := appendContainerArgs(raw, "atenet-router", "atenet-router", args...) + if err != nil { + return nil, err + } + if extra := e.Cfg.RouterParkedRequestBudget - routerManifestParkedRequestBudget; extra > 0 { + grace := routerManifestGracePeriodSeconds + int64(math.Ceil(extra.Seconds())) + raw, err = patchDeployment(raw, "atenet-router", fmt.Sprintf(`- op: replace + path: /spec/template/spec/terminationGracePeriodSeconds + value: %d`, grace)) + } + return raw, err +} + +// appendContainerArgs appends args to the first container of the named +// Deployment in ate-system. The patch first tests that the first container is +// the one named, so a reordered manifest fails the render instead of patching +// the wrong container. func appendContainerArgs(raw []byte, deployment, container string, args ...string) ([]byte, error) { + ops := []string{fmt.Sprintf(`- op: test + path: /spec/template/spec/containers/0/name + value: %q`, container)} + for _, arg := range args { + ops = append(ops, fmt.Sprintf(`- op: add + path: /spec/template/spec/containers/0/args/- + value: %q`, arg)) + } + return patchDeployment(raw, deployment, ops...) +} + +// patchDeployment applies ops, the entries of a JSON6902 patch in YAML, to the +// named Deployment in ate-system through a scratch kustomization. +func patchDeployment(raw []byte, deployment string, ops ...string) ([]byte, error) { dir, err := os.MkdirTemp("", "ate-setup-kustomize-") if err != nil { return nil, fmt.Errorf("while creating a scratch kustomization: %w", err) @@ -334,15 +393,11 @@ patches: name: %s namespace: ate-system patch: |- - - op: test - path: /spec/template/spec/containers/0/name - value: %q -`, deployment, container) - for _, arg := range args { - fmt.Fprintf(&kustomization, ` - op: add - path: /spec/template/spec/containers/0/args/- - value: %q -`, arg) +`, deployment) + for _, op := range ops { + for line := range strings.SplitSeq(op, "\n") { + kustomization.WriteString(" " + line + "\n") + } } if err := os.WriteFile(filepath.Join(dir, "kustomization.yaml"), []byte(kustomization.String()), 0o600); err != nil { return nil, fmt.Errorf("while writing a scratch kustomization: %w", err) diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index ab735825d3..9858f77d37 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -970,6 +970,78 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { } } +// Both paths that deliver the Envoy router pass it the atenet.router.* +// settings, and a parked request budget above the manifest's grows the pod's +// grace period by the difference. Unset, the manifest comes out as checked in, +// so it must not set the flags itself; that case also pins +// routerManifestGracePeriodSeconds to the manifest. +func TestRenderRouterSettings(t *testing.T) { + src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} + resolver := images.NewPrebuilt(src, func(context.Context, string) (string, error) { + return "sha256:" + strings.Repeat("2", 64), nil + }) + flags := []string{"--route-timeout=", "--parked-request-budget="} + for _, tc := range []struct { + name string + cfg config.Config + wantArgs []string + wantGrace int64 + }{ + {name: "unset", wantGrace: routerManifestGracePeriodSeconds}, + { + name: "route timeout and budget", + cfg: config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute}, + wantArgs: []string{"--route-timeout=30m0s", "--parked-request-budget=2m0s"}, + wantGrace: routerManifestGracePeriodSeconds + 115, + }, + { + name: "budget at the router default", + cfg: config.Config{RouterParkedRequestBudget: 5 * time.Second}, + wantArgs: []string{"--parked-request-budget=5s"}, + wantGrace: routerManifestGracePeriodSeconds, + }, + } { + for _, path := range []struct { + name string + kind bool + render func(e *Env) ([]byte, error) + }{ + {name: "base bundle", render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }}, + {name: "kind bundle", kind: true, render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }}, + {name: "router file", render: func(e *Env) ([]byte, error) { return e.renderAtenetRouterManifest(t.Context()) }}, + } { + t.Run(tc.name+"/"+path.name, func(t *testing.T) { + cfg := tc.cfg + cfg.Root = repoRoot(t) + cfg.Router = config.RouterEnvoy + cfg.Kind = path.kind + cfg.Images = src + rendered, err := path.render(&Env{Cfg: &cfg, resolver: resolver}) + if err != nil { + t.Fatalf("render: %v", err) + } + var got []string + for _, arg := range routerArgs(t, rendered) { + if slices.ContainsFunc(flags, func(flag string) bool { return strings.HasPrefix(arg, flag) }) { + got = append(got, arg) + } + } + if !slices.Equal(got, tc.wantArgs) { + t.Errorf("atenet-router settings args = %q, want %q", got, tc.wantArgs) + } + _, deployments := dataPlaneWorkloads(t, rendered) + grace := deployments["atenet-router"].Spec.Template.Spec.TerminationGracePeriodSeconds + if grace == nil { + t.Fatal("the atenet-router pod spec has no terminationGracePeriodSeconds") + } + if *grace != tc.wantGrace { + t.Errorf("atenet-router terminationGracePeriodSeconds = %d, want %d", *grace, tc.wantGrace) + } + }) + } + } +} + func TestRenderAgentgatewayStaticMTLS(t *testing.T) { const ( idA = "spiffe://cluster.local/ns/a/sa/a" diff --git a/docs/concepts/request-parking.md b/docs/concepts/request-parking.md index 6ebb9d24d0..9d0559f749 100644 --- a/docs/concepts/request-parking.md +++ b/docs/concepts/request-parking.md @@ -157,6 +157,17 @@ so a parked request always gets its full budget and a normal verdict (routed The retry backoff deliberately has no cap and no attempt limit: the budget alone bounds the wait. +### Sizing for agent workloads + +The defaults suit short interactive turns. For agents that run long commands, +or pools that are often full, set the budget to the longest wait for a slot +you accept before a `503`, and the route timeout above the longest request an +actor serves. Both are install settings, `--atenet-parked-request-budget` and +`--atenet-route-timeout` (`ATE_ATENET_PARKED_REQUEST_BUDGET`, +`ATE_ATENET_ROUTE_TIMEOUT`); the installer grows the router's +`terminationGracePeriodSeconds` with the budget so parked requests still +finish during a rollout. + ## Observability **Metrics** (OpenTelemetry, meter `atenet-router`): diff --git a/docs/examples/substrate-install.yaml b/docs/examples/substrate-install.yaml index 6fe0af7825..ce9ecc4937 100644 --- a/docs/examples/substrate-install.yaml +++ b/docs/examples/substrate-install.yaml @@ -39,6 +39,10 @@ rolloutTimeout: 1m0s # Ingress and egress dataplane: envoy or agentgateway. atenet: dataplane: envoy + # Router timeouts for agent workloads; unset keeps the router defaults (5s, 5m). Envoy only. + router: + parkedRequestBudget: 2m + routeTimeout: 30m # Prebuilt images. Both halves are required together: a tag with no repository # would be dropped, leaving a build from source that looks like the release the diff --git a/docs/guides/operator-install.md b/docs/guides/operator-install.md index c62050a5c0..d1de2477ad 100644 --- a/docs/guides/operator-install.md +++ b/docs/guides/operator-install.md @@ -300,6 +300,8 @@ demos, the [benchmarks](benchmarking.md#settings), and the | `atenet.dataplane` | `ATE_ATENET_DATAPLANE` | `--atenet-dataplane` | `envoy` | (global) | | `atenet.egress.additionalExtprocService` | `ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE` | `--experimental-additional-egress-extproc-service` | — | (global) | | `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) | +| `atenet.router.parkedRequestBudget` | `ATE_ATENET_PARKED_REQUEST_BUDGET` | `--atenet-parked-request-budget` | — | (global) | +| `atenet.router.routeTimeout` | `ATE_ATENET_ROUTE_TIMEOUT` | `--atenet-route-timeout` | — | (global) | | `clusterSize` | `ATE_INSTALL_CLUSTER_SIZE` | `--cluster-size` | `size0` | (global) | | `context` | `KUBECTL_CONTEXT` | `--context` | — | (global) | | `cordonSystemInfraComponents` | `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS` | `--cordon-system-infra-components` | `false` | (global) | diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 35b630c470..81b9dab83b 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -244,6 +244,8 @@ spec: # default of 5m, which suits an agent relaying a model completion: the # request stays open for the whole generation. Set --route-timeout # lower to cap how long one turn may hold a request open. + # ate-setup appends --route-timeout and --parked-request-budget here from + # its atenet.router.* settings and grows the grace period above with the budget. # # A turn still running after the drain route budget (10s) does not # survive a shutdown, and raising --route-timeout does not change that: From 1051b7501b4a8d706e99eccc9f1e0fcb8c044385 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 13 Oct 2026 09:50:12 -0400 Subject: [PATCH 2/2] ate-setup: setting for the router's ext_proc circuit breaker ATE_ATENET_EXTPROC_MAX_REQUESTS (atenet.router.extprocMaxRequests, --atenet-extproc-max-requests) passes --extproc-max-requests to the router the way the route timeout and parked budget settings pass theirs. On a dense host 1536 synchronized execs overflowed the derived breaker of 2048 and Envoy answered 503 "reset reason: overflow"; the only way to raise it was a patch on the deployment. Signed-off-by: Davanum Srinivas --- cmd/ate-setup/commands.md | 1 + cmd/ate-setup/internal/config/config.go | 3 + cmd/ate-setup/internal/config/config_test.go | 1 + cmd/ate-setup/internal/config/load.go | 14 ++++- cmd/ate-setup/internal/config/load_test.go | 65 +++++++++++++------- cmd/ate-setup/internal/config/setting.go | 5 ++ cmd/ate-setup/internal/steps/overlay.go | 3 + cmd/ate-setup/internal/steps/overlay_test.go | 14 +++-- docs/concepts/request-parking.md | 7 +++ docs/examples/substrate-install.yaml | 6 +- docs/guides/operator-install.md | 1 + manifests/ate-install/atenet-router.yaml | 5 +- 12 files changed, 92 insertions(+), 33 deletions(-) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index d24826f26e..a2f71d5278 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -22,6 +22,7 @@ every setting with its environment variable. | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | | `--atenet-route-timeout DURATION` | The router's `--route-timeout` (default `5m`). Envoy dataplane only. `ATE_ATENET_ROUTE_TIMEOUT` when the flag is absent | | `--atenet-parked-request-budget DURATION` | The router's `--parked-request-budget` (default `5s`); the router pod's `terminationGracePeriodSeconds` grows with it. Envoy dataplane only. `ATE_ATENET_PARKED_REQUEST_BUDGET` when the flag is absent | +| `--atenet-extproc-max-requests N` | The router's `--extproc-max-requests`, the circuit breaker on Envoy's ext_proc cluster (default: twice `--parked-request-max`, at least `1024`). Envoy dataplane only. `ATE_ATENET_EXTPROC_MAX_REQUESTS` when the flag is absent | | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). It also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 1e16efbad5..f38dbd0249 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -171,6 +171,9 @@ type Config struct { // --route-timeout and --parked-request-budget; zero keeps the router default. RouterRouteTimeout time.Duration RouterParkedRequestBudget time.Duration + // RouterExtProcMaxRequests is the router's --extproc-max-requests; zero + // keeps the router's derivation from --parked-request-max. + RouterExtProcMaxRequests int // The read/write and owner connections can use different login identities. // With one configured connection, both pools use it. Both empty selects // bundled PostgreSQL. diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 2e81d022ad..f637d7f6e4 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -60,6 +60,7 @@ func loadEnv(t *testing.T) { "ATE_API_POSTGRES_SCHEMA", "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", + "ATE_ATENET_EXTPROC_MAX_REQUESTS", "ATE_ATENET_PARKED_REQUEST_BUDGET", "ATE_ATENET_ROUTE_TIMEOUT", "ATE_CREDENTIAL_PROVIDER", diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 610932e5e4..3d18e73609 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -241,6 +241,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro Router: r.String("atenet.dataplane"), RouterRouteTimeout: r.Duration("atenet.router.routeTimeout"), RouterParkedRequestBudget: r.Duration("atenet.router.parkedRequestBudget"), + RouterExtProcMaxRequests: r.Int("atenet.router.extprocMaxRequests"), PostgresSchema: r.String("ateapi.postgres.schema"), PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), @@ -397,13 +398,20 @@ func validateResolved(cfg *Config, r *Resolved) error { } // Only the envoy dataplane runs the atenet-router container the flags go on. - for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget"} { + for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget", "atenet.router.extprocMaxRequests"} { v, _ := r.Value(key) if v.Raw == "" { continue } - if r.Duration(key) <= 0 { - return &InvalidError{Value: v, Want: "a positive duration"} + switch v.Setting.Kind { + case KindDuration: + if r.Duration(key) <= 0 { + return &InvalidError{Value: v, Want: "a positive duration"} + } + case KindInt: + if r.Int(key) <= 0 { + return &InvalidError{Value: v, Want: "a positive integer"} + } } if cfg.Router != RouterEnvoy { return &ConflictError{A: v, B: dataplane, Why: "the router flags need the envoy dataplane"} diff --git a/cmd/ate-setup/internal/config/load_test.go b/cmd/ate-setup/internal/config/load_test.go index 2e2c2e5f4b..fec8977cb6 100644 --- a/cmd/ate-setup/internal/config/load_test.go +++ b/cmd/ate-setup/internal/config/load_test.go @@ -228,6 +228,7 @@ func TestLoadFlagsRouterSettings(t *testing.T) { loadEnv(t) t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "30m") t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + t.Setenv("ATE_ATENET_EXTPROC_MAX_REQUESTS", "4096") fs, err := optionsFlagSet(Options{}) if err != nil { @@ -243,37 +244,55 @@ func TestLoadFlagsRouterSettings(t *testing.T) { if cfg.RouterParkedRequestBudget != 2*time.Minute { t.Errorf("RouterParkedRequestBudget = %s, want 2m", cfg.RouterParkedRequestBudget) } + if cfg.RouterExtProcMaxRequests != 4096 { + t.Errorf("RouterExtProcMaxRequests = %d, want 4096", cfg.RouterExtProcMaxRequests) + } } func TestLoadFlagsRouterSettingsNeedEnvoy(t *testing.T) { - loadEnv(t) - t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway) - t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + for env, value := range map[string]string{ + "ATE_ATENET_PARKED_REQUEST_BUDGET": "2m", + "ATE_ATENET_EXTPROC_MAX_REQUESTS": "4096", + } { + t.Run(env, func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway) + t.Setenv(env, value) - fs, err := optionsFlagSet(Options{}) - if err != nil { - t.Fatal(err) - } - _, err = LoadFlags(fs, LoadOptions{}) - if err == nil { - t.Fatal("LoadFlags() = nil error, want a dataplane conflict") - } - for _, want := range []string{"ATE_ATENET_PARKED_REQUEST_BUDGET", "ATE_ATENET_DATAPLANE"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("error %q does not name %s", err, want) - } + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + _, err = LoadFlags(fs, LoadOptions{}) + if err == nil { + t.Fatal("LoadFlags() = nil error, want a dataplane conflict") + } + for _, want := range []string{env, "ATE_ATENET_DATAPLANE"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %s", err, want) + } + } + }) } } func TestLoadFlagsRouterSettingsMustBePositive(t *testing.T) { - loadEnv(t) - t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "0s") + for _, tc := range []struct{ env, value, want string }{ + {"ATE_ATENET_ROUTE_TIMEOUT", "0s", "positive duration"}, + {"ATE_ATENET_EXTPROC_MAX_REQUESTS", "0", "positive integer"}, + {"ATE_ATENET_EXTPROC_MAX_REQUESTS", "-1", "positive integer"}, + } { + t.Run(tc.env+"="+tc.value, func(t *testing.T) { + loadEnv(t) + t.Setenv(tc.env, tc.value) - fs, err := optionsFlagSet(Options{}) - if err != nil { - t.Fatal(err) - } - if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), "positive duration") { - t.Errorf("LoadFlags() error = %v, want a positive-duration error", err) + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("LoadFlags() error = %v, want a %s error", err, tc.want) + } + }) } } diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 0c76802b65..caac70ce73 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -174,6 +174,11 @@ var Registry = []Setting{ Kind: KindDuration, Usage: "The router's --parked-request-budget; unset keeps the router default; the router pod's grace period grows with it (envoy dataplane only)", }, + { + Key: "atenet.router.extprocMaxRequests", Env: "ATE_ATENET_EXTPROC_MAX_REQUESTS", Flag: "atenet-extproc-max-requests", + Kind: KindInt, + Usage: "The router's --extproc-max-requests, the circuit breaker on Envoy's ext_proc cluster; unset keeps the router's derivation (envoy dataplane only)", + }, { Key: "atenet.egress.additionalExtprocService", Env: "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", Flag: "experimental-additional-egress-extproc-service", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 0ea1134d09..fc4f7fcb49 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -336,6 +336,9 @@ func (e *Env) patchAtenetRouterSettings(raw []byte) ([]byte, error) { if d := e.Cfg.RouterParkedRequestBudget; d > 0 { args = append(args, "--parked-request-budget="+d.String()) } + if n := e.Cfg.RouterExtProcMaxRequests; n > 0 { + args = append(args, "--extproc-max-requests="+strconv.Itoa(n)) + } if len(args) == 0 { return raw, nil } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 9858f77d37..ec4b210580 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -980,7 +980,7 @@ func TestRenderRouterSettings(t *testing.T) { resolver := images.NewPrebuilt(src, func(context.Context, string) (string, error) { return "sha256:" + strings.Repeat("2", 64), nil }) - flags := []string{"--route-timeout=", "--parked-request-budget="} + flags := []string{"--route-timeout=", "--parked-request-budget=", "--extproc-max-requests="} for _, tc := range []struct { name string cfg config.Config @@ -989,9 +989,9 @@ func TestRenderRouterSettings(t *testing.T) { }{ {name: "unset", wantGrace: routerManifestGracePeriodSeconds}, { - name: "route timeout and budget", - cfg: config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute}, - wantArgs: []string{"--route-timeout=30m0s", "--parked-request-budget=2m0s"}, + name: "all set", + cfg: config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute, RouterExtProcMaxRequests: 4096}, + wantArgs: []string{"--route-timeout=30m0s", "--parked-request-budget=2m0s", "--extproc-max-requests=4096"}, wantGrace: routerManifestGracePeriodSeconds + 115, }, { @@ -1000,6 +1000,12 @@ func TestRenderRouterSettings(t *testing.T) { wantArgs: []string{"--parked-request-budget=5s"}, wantGrace: routerManifestGracePeriodSeconds, }, + { + name: "breaker alone", + cfg: config.Config{RouterExtProcMaxRequests: 4096}, + wantArgs: []string{"--extproc-max-requests=4096"}, + wantGrace: routerManifestGracePeriodSeconds, + }, } { for _, path := range []struct { name string diff --git a/docs/concepts/request-parking.md b/docs/concepts/request-parking.md index 9d0559f749..721d181aeb 100644 --- a/docs/concepts/request-parking.md +++ b/docs/concepts/request-parking.md @@ -168,6 +168,13 @@ actor serves. Both are install settings, `--atenet-parked-request-budget` and `terminationGracePeriodSeconds` with the budget so parked requests still finish during a rollout. +A burst of requests larger than the ext_proc circuit breaker is rejected by +Envoy itself, with `503` and `reset reason: overflow`, before the router sees +it. The router derives the breaker as twice `--parked-request-max` (at least +`1024`); the install setting `--atenet-extproc-max-requests` +(`ATE_ATENET_EXTPROC_MAX_REQUESTS`) raises it for hosts dense enough to +synchronize more requests than that. + ## Observability **Metrics** (OpenTelemetry, meter `atenet-router`): diff --git a/docs/examples/substrate-install.yaml b/docs/examples/substrate-install.yaml index ce9ecc4937..4c2b0ec77f 100644 --- a/docs/examples/substrate-install.yaml +++ b/docs/examples/substrate-install.yaml @@ -39,10 +39,14 @@ rolloutTimeout: 1m0s # Ingress and egress dataplane: envoy or agentgateway. atenet: dataplane: envoy - # Router timeouts for agent workloads; unset keeps the router defaults (5s, 5m). Envoy only. + # Router settings for agent workloads; unset keeps the router defaults. Envoy only. router: + # Park budget and route timeout; the router defaults are 5s and 5m. parkedRequestBudget: 2m routeTimeout: 30m + # Circuit breaker on Envoy's ext_proc cluster; the router derives twice the + # parking lot (at least 1024) when unset. + extprocMaxRequests: 4096 # Prebuilt images. Both halves are required together: a tag with no repository # would be dropped, leaving a build from source that looks like the release the diff --git a/docs/guides/operator-install.md b/docs/guides/operator-install.md index d1de2477ad..bc6ae91c4f 100644 --- a/docs/guides/operator-install.md +++ b/docs/guides/operator-install.md @@ -300,6 +300,7 @@ demos, the [benchmarks](benchmarking.md#settings), and the | `atenet.dataplane` | `ATE_ATENET_DATAPLANE` | `--atenet-dataplane` | `envoy` | (global) | | `atenet.egress.additionalExtprocService` | `ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE` | `--experimental-additional-egress-extproc-service` | — | (global) | | `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) | +| `atenet.router.extprocMaxRequests` | `ATE_ATENET_EXTPROC_MAX_REQUESTS` | `--atenet-extproc-max-requests` | — | (global) | | `atenet.router.parkedRequestBudget` | `ATE_ATENET_PARKED_REQUEST_BUDGET` | `--atenet-parked-request-budget` | — | (global) | | `atenet.router.routeTimeout` | `ATE_ATENET_ROUTE_TIMEOUT` | `--atenet-route-timeout` | — | (global) | | `clusterSize` | `ATE_INSTALL_CLUSTER_SIZE` | `--cluster-size` | `size0` | (global) | diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 81b9dab83b..2a7ffdf588 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -244,8 +244,9 @@ spec: # default of 5m, which suits an agent relaying a model completion: the # request stays open for the whole generation. Set --route-timeout # lower to cap how long one turn may hold a request open. - # ate-setup appends --route-timeout and --parked-request-budget here from - # its atenet.router.* settings and grows the grace period above with the budget. + # ate-setup appends --route-timeout, --parked-request-budget and + # --extproc-max-requests here from its atenet.router.* settings and grows + # the grace period above with the budget. # # A turn still running after the drain route budget (10s) does not # survive a shutdown, and raising --route-timeout does not change that: