Egress policies can inject tokens from RFC 8693 token exchange

api/ateapi/v1/ateapi.proto

@@ -621,7 +621,7 @@ message EgressPolicy {
   // +k8s:optional
   // +k8s:maxItems=256
   // +k8s:listType=atomic
-  // +k8s:customValidation # no two rules tie on a name and port
+  // +k8s:customValidation # no two rules tie on a name and port, and an https rule allows each token_url
   repeated EgressRule rules = 2;
 }
 
@@ -636,7 +636,7 @@ message EgressPolicyTemplate {
   // +k8s:required
   // +k8s:maxItems=256
   // +k8s:listType=atomic
-  // +k8s:customValidation # no two rules tie on a name and port
+  // +k8s:customValidation # no two rules tie on a name and port, and an https rule allows each token_url
   repeated EgressRule rules = 1;
 }
 
@@ -839,7 +839,7 @@ message CredentialHeader {
   // Source-agnostic reference interpreted by a registered credential provider:
   // `ate-secret://<provider-class>/<provider-name>/<provider-specific-tail>`
   //
-  // Exactly one of credential_uri and actor_jwt must be set.
+  // Exactly one of credential_uri, actor_jwt, and token_exchange must be set.
   //
   // +k8s:optional
   // +k8s:unionMember
@@ -852,6 +852,45 @@ message CredentialHeader {
   // +k8s:optional
   // +k8s:unionMember
   ActorJwtSource actor_jwt = 4;
+
+  // An access token obtained by exchanging an actor JWT at an OAuth 2.0
+  // token exchange endpoint (RFC 8693). A policy can have at most 16
+  // token_exchange entries.
+  //
+  // +k8s:optional
+  // +k8s:unionMember
+  TokenExchangeSource token_exchange = 5;
+}
+
+// TokenExchangeSource has the gateway mint a JWT for the Actor that sent the
+// request, bound to audience, and exchange it at token_url (RFC 8693). The
+// returned access token becomes the header value.
+message TokenExchangeSource {
+  // HTTPS URL of the token endpoint, with a DNS hostname and without userinfo,
+  // query, or fragment. An https rule in the same policy must allow its host
+  // and port.
+  //
+  // +k8s:required
+  // +k8s:maxLength=2048
+  // +k8s:customValidation # format
+  string token_url = 1;
+
+  // Becomes the actor JWT's only aud and the request's audience parameter.
+  //
+  // +k8s:required
+  // +k8s:minLength=1
+  // +k8s:maxLength=512
+  string audience = 2;
+
+  // The request's scope parameter, space-joined.
+  //
+  // +k8s:optional
+  // +k8s:maxItems=16
+  // +k8s:listType=set
+  // +k8s:eachVal=+k8s:minLength=1
+  // +k8s:eachVal=+k8s:maxLength=256
+  // +k8s:customValidation # RFC 6749 scope-token characters
+  repeated string scopes = 3;
 }
 
 // ActorJwtSource has the gateway mint a JWT for the Actor that sent the

cmd/ate-setup/commands.md

@@ -30,6 +30,7 @@ every setting with its environment variable.
 | `--egress-gateway-extra-spiffe-ids IDS` | Comma-separated SPIFFE IDs that hold ate-api-server's `egress_gateway` role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress's ID and then each listed ID as repeated `--egress-gateway-spiffe-id` flags; unset passes none, leaving ate-api-server's default. `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` when the flag is absent |
 | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter |
 | `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"<provider>","address":"<host>:<port>"}` uses a provider you deploy yourself. A JSON array of provider objects selects several, each serving the `ate-secret://` URIs its name is the host of. See [`docs/guides/egress-credential-injection.md`](../../docs/guides/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent |
+| `--token-exchange-ca-configmap NAME` | ConfigMap in the install namespace whose `ca.crt` key holds PEM certificates the egress gateway trusts for token exchange endpoints, in addition to the system roots. The installer mounts it but does not create it: `deploy ate-system` and `deploy atenet` fail if it or its `ca.crt` key is missing. The gateway reads it at startup, so restart `atenet-egress` after changing it. Requires `--atenet-dataplane=envoy`. `ATE_TOKEN_EXCHANGE_CA_CONFIGMAP` when the flag is absent |
 | `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). Changing the level updates the env-hash annotation and rolls `ate-api-server`. `ATE_API_LOG_LEVEL` when the flag is absent |
 | `--otlp-endpoint URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)). `ATE_OTLP_ENDPOINT` when the flag is absent |
 | `--preview GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set |

cmd/ate-setup/internal/config/config.go

@@ -242,6 +242,10 @@ type Config struct {
 	// CredentialProviders for its schema.
 	CredentialProviderJSON string
 
+	// TokenExchangeCAConfigMap names the admin-owned ConfigMap whose ca.crt
+	// the egress gateway trusts for token exchange endpoints.
+	TokenExchangeCAConfigMap string
+
 	// AnthropicAPIKey is required only by the claude-code-multiplex demo.
 	AnthropicAPIKey string
 

cmd/ate-setup/internal/config/config_test.go

@@ -74,6 +74,7 @@ func loadEnv(t *testing.T) {
 		"ATE_INSTALL_ROLLOUT_TIMEOUT",
 		"ATE_OTLP_ENDPOINT",
 		"ATE_PREVIEW",
+		"ATE_TOKEN_EXCHANGE_CA_CONFIGMAP",
 		"BENCHMARK_ACTOR_MEMORY",
 		"BUCKET_NAME",
 		"CLUSTER_LOCATION",
@@ -1042,6 +1043,8 @@ func TestLoadRejectsInvalidValues(t *testing.T) {
 		{"cluster size", Options{ClusterSize: "size5"}},
 		{"extproc invalid format", Options{AdditionalEgressExtprocService: "extproc:50051"}},
 		{"extproc agentgateway", Options{Router: RouterAgentgateway, AdditionalEgressExtprocService: "ate-system/extproc:50051"}},
+		{"token exchange CA name", Options{TokenExchangeCAConfigMap: "Corp_CA"}},
+		{"token exchange CA agentgateway", Options{Router: RouterAgentgateway, TokenExchangeCAConfigMap: "corp-ca"}},
 		{"provider not JSON", Options{CredentialProvider: "k8s.io"}},
 		{"provider as a JSON string", Options{CredentialProvider: `"off"`}},
 		{"provider null", Options{CredentialProvider: `null`}},

cmd/ate-setup/internal/config/errors_test.go

@@ -209,6 +209,11 @@ func TestValidationErrorsNameTheSupplyingChannel(t *testing.T) {
 			env:    map[string]string{"ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE": "ns/svc:99999"},
 			wantIn: []string{"atenet.egress.additionalExtprocService", "1-65535", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE"},
 		},
+		{
+			name:   "token exchange CA is not a ConfigMap name",
+			env:    map[string]string{"ATE_TOKEN_EXCHANGE_CA_CONFIGMAP": "Corp_CA"},
+			wantIn: []string{"atenet.egress.tokenExchangeCAConfigMap", "a ConfigMap name", `"Corp_CA"`, "ATE_TOKEN_EXCHANGE_CA_CONFIGMAP"},
+		},
 		{
 			name:   "image tag without a repository",
 			env:    map[string]string{"ATE_IMAGE_TAG": "v1"},

cmd/ate-setup/internal/config/load.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/spf13/pflag"
 	"github.com/spiffe/go-spiffe/v2/spiffeid"
+	"k8s.io/apimachinery/pkg/util/validation"
 
 	"go.ate.dev/substrate/cmd/ate-setup/internal/images"
 	"go.ate.dev/substrate/internal/actoridjwt"
@@ -273,6 +274,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro
 		IngressAllowedSPIFFEIDs:        splitList(r.String("ingressAllowedSPIFFEIDs")),
 		AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"),
 		CredentialProviderJSON:         r.String("atenet.egress.credentialProvider"),
+		TokenExchangeCAConfigMap:       r.String("atenet.egress.tokenExchangeCAConfigMap"),
 		AnthropicAPIKey:                r.String("demo.anthropicAPIKey"),
 		OtlpEndpoint:                   r.String("otlpEndpoint"),
 		APILogLevel:                    r.String("ateapi.logLevel"),
@@ -449,6 +451,16 @@ func validateResolved(cfg *Config, r *Resolved) error {
 				Why: "an additional ext_proc filter requires dataplane " + RouterEnvoy}
 		}
 	}
+	if cfg.TokenExchangeCAConfigMap != "" {
+		ca, _ := r.Value("atenet.egress.tokenExchangeCAConfigMap")
+		if len(validation.IsDNS1123Subdomain(cfg.TokenExchangeCAConfigMap)) != 0 {
+			return &InvalidError{Value: ca, Want: "a ConfigMap name"}
+		}
+		if cfg.Router != RouterEnvoy {
+			return &ConflictError{A: ca, B: dataplane,
+				Why: "token exchange requires dataplane " + RouterEnvoy}
+		}
+	}
 	// A gate the components do not know makes each of them refuse to start, so
 	// catch it here rather than as a failed rollout.
 	if err := preview.Verify(cfg.PreviewGates...); err != nil {

cmd/ate-setup/internal/config/options_test.go

@@ -43,6 +43,7 @@ type Options struct {
 	E2ETuning                      bool
 	AdditionalEgressExtprocService string
 	CredentialProvider             string
+	TokenExchangeCAConfigMap       string
 	OtlpEndpoint                   string
 
 	// Image source selection.
@@ -91,6 +92,7 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) {
 		{"cluster-size", opts.ClusterSize},
 		{"experimental-additional-egress-extproc-service", opts.AdditionalEgressExtprocService},
 		{"credential-provider", opts.CredentialProvider},
+		{"token-exchange-ca-configmap", opts.TokenExchangeCAConfigMap},
 		{"otlp-endpoint", opts.OtlpEndpoint},
 		{"image-repo", opts.ImageRepo},
 		{"image-tag", opts.ImageTag},

cmd/ate-setup/internal/config/setting.go

@@ -169,6 +169,13 @@ var Registry = []Setting{
 		Usage: "Run an additional ext_proc authorization filter served by NS/SVC:PORT " +
 			"(requires --atenet-dataplane=envoy)",
 	},
+	{
+		Key: "atenet.egress.tokenExchangeCAConfigMap", Env: "ATE_TOKEN_EXCHANGE_CA_CONFIGMAP",
+		Flag: "token-exchange-ca-configmap", Kind: KindString,
+		Usage: "ConfigMap in the install namespace whose ca.crt key holds PEM certificates the egress " +
+			"gateway trusts for token exchange endpoints, besides the system roots. The installer does " +
+			"not create it (requires --atenet-dataplane=envoy)",
+	},
 
 	{
 		// No default: an unset level keeps the one an existing install has,

cmd/ate-setup/internal/steps/deploy.go

@@ -97,6 +97,9 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error {
 	if err := e.requireNoWorkerPods(ctx); err != nil {
 		return err
 	}
+	if err := e.requireTokenExchangeCA(ctx); err != nil {
+		return err
+	}
 	var microvmPrep *microVMPrepared
 	if microvm {
 		microvmPrep, err = e.prepareMicroVMDeps(ctx)
@@ -556,6 +559,9 @@ func (e *Env) DeployAtenet(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	if err := e.requireTokenExchangeCA(ctx); err != nil {
+		return err
+	}
 	if err := e.EnsureCRDs(ctx); err != nil {
 		return err
 	}

cmd/ate-setup/internal/steps/overlay.go

@@ -275,6 +275,10 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, providers []config
 	if err != nil {
 		return nil, err
 	}
+	raw, err = patchAtenetEgressTokenExchangeCA(raw, e.Cfg.TokenExchangeCAConfigMap)
+	if err != nil {
+		return nil, err
+	}
 	raw = e.patchEnvoyDataplaneImage(raw, imageReference)
 	if e.Cfg.EnvoyConcurrency > 0 {
 		raw, err = patchAtenetEgressEnvoyConcurrency(raw, e.Cfg.EnvoyConcurrency)
@@ -389,6 +393,65 @@ func patchAgentgatewayEgressInject(raw []byte, providers []config.CredentialProv
 	return replaceManifestMarker(raw, "#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS", block)
 }
 
+const (
+	tokenExchangeCADir = "/run/token-exchange-ca"
+	tokenExchangeCAKey = "ca.crt"
+)
+
+// patchAtenetEgressTokenExchangeCA replaces the token exchange CA markers in
+// the egress sidecar with --token-exchange-ca-file and a mount of configMap's
+// ca.crt, or removes them when configMap is empty.
+func patchAtenetEgressTokenExchangeCA(raw []byte, configMap string) ([]byte, error) {
+	var flag, mount, volume string
+	if configMap != "" {
+		flag = fmt.Sprintf("- --token-exchange-ca-file=%s/%s", tokenExchangeCADir, tokenExchangeCAKey)
+		mount = fmt.Sprintf(`- name: token-exchange-ca
+  mountPath: %s
+  readOnly: true`, tokenExchangeCADir)
+		volume = fmt.Sprintf(`- name: token-exchange-ca
+  configMap:
+    name: %q
+    items:
+    - key: %s
+      path: %s`, configMap, tokenExchangeCAKey, tokenExchangeCAKey)
+	}
+	for _, splice := range []struct{ marker, block string }{
+		{"#ATE_TOKEN_EXCHANGE_CA_FLAG", flag},
+		{"#ATE_TOKEN_EXCHANGE_CA_MOUNT", mount},
+		{"#ATE_TOKEN_EXCHANGE_CA_VOLUME", volume},
+	} {
+		var err error
+		if raw, err = replaceManifestMarker(raw, splice.marker, splice.block); err != nil {
+			return nil, err
+		}
+	}
+	return raw, nil
+}
+
+// requireTokenExchangeCA fails when --token-exchange-ca-configmap names a
+// ConfigMap the install namespace lacks, or one without a ca.crt key, which
+// would leave the egress gateway unable to start.
+func (e *Env) requireTokenExchangeCA(ctx context.Context) error {
+	name := e.Cfg.TokenExchangeCAConfigMap
+	if name == "" {
+		return nil
+	}
+	cm, err := e.Kube.GetConfigMap(ctx, e.Namespace(), name)
+	if err != nil {
+		return err
+	}
+	if cm == nil {
+		return fmt.Errorf("token exchange CA ConfigMap %s/%s does not exist; create it with the CA certificates under key %s",
+			e.Namespace(), name, tokenExchangeCAKey)
+	}
+	_, inData := cm.Data[tokenExchangeCAKey]
+	_, inBinary := cm.BinaryData[tokenExchangeCAKey]
+	if !inData && !inBinary {
+		return fmt.Errorf("token exchange CA ConfigMap %s/%s has no %s key", e.Namespace(), name, tokenExchangeCAKey)
+	}
+	return nil
+}
+
 func replaceManifestMarker(raw []byte, marker, block string) ([]byte, error) {
 	var out []string
 	replaced := 0

cmd/ate-setup/internal/steps/overlay_test.go

@@ -1623,6 +1623,105 @@ func TestRenderAtenetEgressManifestEnvoyConcurrency(t *testing.T) {
 	})
 }
 
+func TestRenderAtenetEgressManifestTokenExchangeCA(t *testing.T) {
+	const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
+	src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"}
+	render := func(t *testing.T, configMap string) (corev1.Container, []corev1.Volume) {
+		t.Helper()
+		e := &Env{
+			Cfg: &config.Config{Root: repoRoot(t), Router: config.RouterEnvoy, Images: src, TokenExchangeCAConfigMap: configMap},
+			resolver: images.NewPrebuilt(src, func(context.Context, string) (string, error) {
+				return digest, nil
+			}),
+		}
+		// No credential provider: the CA does not depend on one.
+		out, err := e.renderAtenetEgressManifest(t.Context(), nil)
+		if err != nil {
+			t.Fatalf("renderAtenetEgressManifest() error = %v", err)
+		}
+		for _, doc := range strings.Split(string(out), "\n---\n") {
+			var dep appsv1.Deployment
+			if err := yaml.Unmarshal([]byte(doc), &dep); err != nil || dep.Kind != "Deployment" || dep.Name != "atenet-egress" {
+				continue
+			}
+			for _, c := range dep.Spec.Template.Spec.Containers {
+				if c.Name == "ext-proc" {
+					return c, dep.Spec.Template.Spec.Volumes
+				}
+			}
+			t.Fatal("atenet-egress Deployment has no ext-proc container")
+		}
+		t.Fatal("rendered manifest has no atenet-egress Deployment")
+		return corev1.Container{}, nil
+	}
+	const flag = "--token-exchange-ca-file=/run/token-exchange-ca/ca.crt"
+
+	t.Run("unset", func(t *testing.T) {
+		extProc, volumes := render(t, "")
+		if slices.Contains(extProc.Args, flag) {
+			t.Errorf("ext-proc args contain %s with no ConfigMap set", flag)
+		}
+		if slices.ContainsFunc(extProc.VolumeMounts, func(m corev1.VolumeMount) bool { return m.Name == "token-exchange-ca" }) {
+			t.Error("ext-proc mounts token-exchange-ca with no ConfigMap set")
+		}
+		if slices.ContainsFunc(volumes, func(v corev1.Volume) bool { return v.Name == "token-exchange-ca" }) {
+			t.Error("atenet-egress has a token-exchange-ca volume with no ConfigMap set")
+		}
+	})
+
+	t.Run("set", func(t *testing.T) {
+		extProc, volumes := render(t, "corp-ca")
+		if !slices.Contains(extProc.Args, flag) {
+			t.Errorf("ext-proc args = %v, want %s", extProc.Args, flag)
+		}
+		wantMount := corev1.VolumeMount{Name: "token-exchange-ca", MountPath: "/run/token-exchange-ca", ReadOnly: true}
+		if !slices.ContainsFunc(extProc.VolumeMounts, func(m corev1.VolumeMount) bool { return reflect.DeepEqual(m, wantMount) }) {
+			t.Errorf("ext-proc volume mounts = %v, want %v", extProc.VolumeMounts, wantMount)
+		}
+		wantVolume := corev1.Volume{Name: "token-exchange-ca", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{Name: "corp-ca"},
+			Items:                []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}},
+		}}}
+		if !slices.ContainsFunc(volumes, func(v corev1.Volume) bool { return reflect.DeepEqual(v, wantVolume) }) {
+			t.Errorf("atenet-egress volumes = %v, want one equal to %v", volumes, wantVolume)
+		}
+	})
+
+	if _, err := patchAtenetEgressTokenExchangeCA([]byte("kind: ConfigMap\n"), "corp-ca"); err == nil {
+		t.Error("patchAtenetEgressTokenExchangeCA accepted a manifest without the markers")
+	}
+}
+
+func TestRequireTokenExchangeCA(t *testing.T) {
+	configMap := func(data, binaryData map[string]string) *corev1.ConfigMap {
+		cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: "corp-ca"}, Data: data}
+		for k, v := range binaryData {
+			cm.BinaryData = map[string][]byte{k: []byte(v)}
+		}
+		return cm
+	}
+	for _, tc := range []struct {
+		name      string
+		configMap string
+		objects   []runtime.Object
+		wantErr   bool
+	}{
+		{name: "unset"},
+		{name: "present", configMap: "corp-ca", objects: []runtime.Object{configMap(map[string]string{"ca.crt": "pem"}, nil)}},
+		{name: "present as binary data", configMap: "corp-ca", objects: []runtime.Object{configMap(nil, map[string]string{"ca.crt": "der"})}},
+		{name: "missing", configMap: "corp-ca", wantErr: true},
+		{name: "without ca.crt", configMap: "corp-ca", objects: []runtime.Object{configMap(map[string]string{"tls.crt": "pem"}, nil)}, wantErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			e := &Env{Cfg: &config.Config{Root: repoRoot(t), TokenExchangeCAConfigMap: tc.configMap}, Kube: fakeKube(t, tc.objects...)}
+			err := e.requireTokenExchangeCA(t.Context())
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("requireTokenExchangeCA() = %v, want error %v", err, tc.wantErr)
+			}
+		})
+	}
+}
+
 // Without --fake-data-plane there is no fake-workersync rollout to wait for.
 func TestWaitFakeWorkersyncWithoutTheFlagIsANoOp(t *testing.T) {
 	e := &Env{Cfg: &config.Config{Root: repoRoot(t)}, Kube: fakeKube(t)}