Skip to content
1,666 changes: 885 additions & 781 deletions api/ateapi/v1/ateapi.pb.go

Large diffs are not rendered by default.

45 changes: 42 additions & 3 deletions api/ateapi/v1/ateapi.proto
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,7 @@ message EgressPolicy {
// +k8s:optional
// +k8s:maxItems=256
// +k8s:listType=atomic
// +k8s:customValidation # no two rules tie on a name and port
// +k8s:customValidation # no two rules tie on a name and port, and an https rule allows each token_url
repeated EgressRule rules = 2;
}

Expand All @@ -636,7 +636,7 @@ message EgressPolicyTemplate {
// +k8s:required
// +k8s:maxItems=256
// +k8s:listType=atomic
// +k8s:customValidation # no two rules tie on a name and port
// +k8s:customValidation # no two rules tie on a name and port, and an https rule allows each token_url
repeated EgressRule rules = 1;
}

Expand Down Expand Up @@ -839,7 +839,7 @@ message CredentialHeader {
// Source-agnostic reference interpreted by a registered credential provider:
// `ate-secret://<provider-class>/<provider-name>/<provider-specific-tail>`
//
// Exactly one of credential_uri and actor_jwt must be set.
// Exactly one of credential_uri, actor_jwt, and token_exchange must be set.
//
// +k8s:optional
// +k8s:unionMember
Expand All @@ -852,6 +852,45 @@ message CredentialHeader {
// +k8s:optional
// +k8s:unionMember
ActorJwtSource actor_jwt = 4;

// An access token obtained by exchanging an actor JWT at an OAuth 2.0
// token exchange endpoint (RFC 8693). A policy can have at most 16
// token_exchange entries.
//
// +k8s:optional
// +k8s:unionMember
TokenExchangeSource token_exchange = 5;
}

// TokenExchangeSource has the gateway mint a JWT for the Actor that sent the
// request, bound to audience, and exchange it at token_url (RFC 8693). The
// returned access token becomes the header value.
message TokenExchangeSource {
// HTTPS URL of the token endpoint, with a DNS hostname and without userinfo,
// query, or fragment. An https rule in the same policy must allow its host
// and port.
//
// +k8s:required
// +k8s:maxLength=2048
// +k8s:customValidation # format
string token_url = 1;

// Becomes the actor JWT's only aud and the request's audience parameter.
//
// +k8s:required
// +k8s:minLength=1
// +k8s:maxLength=512
string audience = 2;

// The request's scope parameter, space-joined.
//
// +k8s:optional
// +k8s:maxItems=16
// +k8s:listType=set
// +k8s:eachVal=+k8s:minLength=1
// +k8s:eachVal=+k8s:maxLength=256
// +k8s:customValidation # RFC 6749 scope-token characters
repeated string scopes = 3;
}

// ActorJwtSource has the gateway mint a JWT for the Actor that sent the
Expand Down
1 change: 1 addition & 0 deletions cmd/ate-setup/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ every setting with its environment variable.
| `--egress-gateway-extra-spiffe-ids IDS` | Comma-separated SPIFFE IDs that hold ate-api-server's `egress_gateway` role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress's ID and then each listed ID as repeated `--egress-gateway-spiffe-id` flags; unset passes none, leaving ate-api-server's default. `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` when the flag is absent |
| `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter |
| `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"<provider>","address":"<host>:<port>"}` uses a provider you deploy yourself. A JSON array of provider objects selects several, each serving the `ate-secret://` URIs its name is the host of. See [`docs/guides/egress-credential-injection.md`](../../docs/guides/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent |
| `--token-exchange-ca-configmap NAME` | ConfigMap in the install namespace whose `ca.crt` key holds PEM certificates the egress gateway trusts for token exchange endpoints, in addition to the system roots. The installer mounts it but does not create it: `deploy ate-system` and `deploy atenet` fail if it or its `ca.crt` key is missing. The gateway reads it at startup, so restart `atenet-egress` after changing it. Requires `--atenet-dataplane=envoy`. `ATE_TOKEN_EXCHANGE_CA_CONFIGMAP` when the flag is absent |
| `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). Changing the level updates the env-hash annotation and rolls `ate-api-server`. `ATE_API_LOG_LEVEL` when the flag is absent |
| `--otlp-endpoint URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)). `ATE_OTLP_ENDPOINT` when the flag is absent |
| `--preview GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set |
Expand Down
4 changes: 4 additions & 0 deletions cmd/ate-setup/internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,10 @@ type Config struct {
// CredentialProviders for its schema.
CredentialProviderJSON string

// TokenExchangeCAConfigMap names the admin-owned ConfigMap whose ca.crt
// the egress gateway trusts for token exchange endpoints.
TokenExchangeCAConfigMap string

// AnthropicAPIKey is required only by the claude-code-multiplex demo.
AnthropicAPIKey string

Expand Down
3 changes: 3 additions & 0 deletions cmd/ate-setup/internal/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ func loadEnv(t *testing.T) {
"ATE_INSTALL_ROLLOUT_TIMEOUT",
"ATE_OTLP_ENDPOINT",
"ATE_PREVIEW",
"ATE_TOKEN_EXCHANGE_CA_CONFIGMAP",
"BENCHMARK_ACTOR_MEMORY",
"BUCKET_NAME",
"CLUSTER_LOCATION",
Expand Down Expand Up @@ -1042,6 +1043,8 @@ func TestLoadRejectsInvalidValues(t *testing.T) {
{"cluster size", Options{ClusterSize: "size5"}},
{"extproc invalid format", Options{AdditionalEgressExtprocService: "extproc:50051"}},
{"extproc agentgateway", Options{Router: RouterAgentgateway, AdditionalEgressExtprocService: "ate-system/extproc:50051"}},
{"token exchange CA name", Options{TokenExchangeCAConfigMap: "Corp_CA"}},
{"token exchange CA agentgateway", Options{Router: RouterAgentgateway, TokenExchangeCAConfigMap: "corp-ca"}},
{"provider not JSON", Options{CredentialProvider: "k8s.io"}},
{"provider as a JSON string", Options{CredentialProvider: `"off"`}},
{"provider null", Options{CredentialProvider: `null`}},
Expand Down
5 changes: 5 additions & 0 deletions cmd/ate-setup/internal/config/errors_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,11 @@ func TestValidationErrorsNameTheSupplyingChannel(t *testing.T) {
env: map[string]string{"ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE": "ns/svc:99999"},
wantIn: []string{"atenet.egress.additionalExtprocService", "1-65535", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE"},
},
{
name: "token exchange CA is not a ConfigMap name",
env: map[string]string{"ATE_TOKEN_EXCHANGE_CA_CONFIGMAP": "Corp_CA"},
wantIn: []string{"atenet.egress.tokenExchangeCAConfigMap", "a ConfigMap name", `"Corp_CA"`, "ATE_TOKEN_EXCHANGE_CA_CONFIGMAP"},
},
{
name: "image tag without a repository",
env: map[string]string{"ATE_IMAGE_TAG": "v1"},
Expand Down
12 changes: 12 additions & 0 deletions cmd/ate-setup/internal/config/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/spf13/pflag"
"github.com/spiffe/go-spiffe/v2/spiffeid"
"k8s.io/apimachinery/pkg/util/validation"

"go.ate.dev/substrate/cmd/ate-setup/internal/images"
"go.ate.dev/substrate/internal/actoridjwt"
Expand Down Expand Up @@ -273,6 +274,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro
IngressAllowedSPIFFEIDs: splitList(r.String("ingressAllowedSPIFFEIDs")),
AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"),
CredentialProviderJSON: r.String("atenet.egress.credentialProvider"),
TokenExchangeCAConfigMap: r.String("atenet.egress.tokenExchangeCAConfigMap"),
AnthropicAPIKey: r.String("demo.anthropicAPIKey"),
OtlpEndpoint: r.String("otlpEndpoint"),
APILogLevel: r.String("ateapi.logLevel"),
Expand Down Expand Up @@ -449,6 +451,16 @@ func validateResolved(cfg *Config, r *Resolved) error {
Why: "an additional ext_proc filter requires dataplane " + RouterEnvoy}
}
}
if cfg.TokenExchangeCAConfigMap != "" {
ca, _ := r.Value("atenet.egress.tokenExchangeCAConfigMap")
if len(validation.IsDNS1123Subdomain(cfg.TokenExchangeCAConfigMap)) != 0 {
return &InvalidError{Value: ca, Want: "a ConfigMap name"}
}
if cfg.Router != RouterEnvoy {
return &ConflictError{A: ca, B: dataplane,
Why: "token exchange requires dataplane " + RouterEnvoy}
}
}
// A gate the components do not know makes each of them refuse to start, so
// catch it here rather than as a failed rollout.
if err := preview.Verify(cfg.PreviewGates...); err != nil {
Expand Down
2 changes: 2 additions & 0 deletions cmd/ate-setup/internal/config/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type Options struct {
E2ETuning bool
AdditionalEgressExtprocService string
CredentialProvider string
TokenExchangeCAConfigMap string
OtlpEndpoint string

// Image source selection.
Expand Down Expand Up @@ -91,6 +92,7 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) {
{"cluster-size", opts.ClusterSize},
{"experimental-additional-egress-extproc-service", opts.AdditionalEgressExtprocService},
{"credential-provider", opts.CredentialProvider},
{"token-exchange-ca-configmap", opts.TokenExchangeCAConfigMap},
{"otlp-endpoint", opts.OtlpEndpoint},
{"image-repo", opts.ImageRepo},
{"image-tag", opts.ImageTag},
Expand Down
7 changes: 7 additions & 0 deletions cmd/ate-setup/internal/config/setting.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,13 @@ var Registry = []Setting{
Usage: "Run an additional ext_proc authorization filter served by NS/SVC:PORT " +
"(requires --atenet-dataplane=envoy)",
},
{
Key: "atenet.egress.tokenExchangeCAConfigMap", Env: "ATE_TOKEN_EXCHANGE_CA_CONFIGMAP",
Flag: "token-exchange-ca-configmap", Kind: KindString,
Usage: "ConfigMap in the install namespace whose ca.crt key holds PEM certificates the egress " +
"gateway trusts for token exchange endpoints, besides the system roots. The installer does " +
"not create it (requires --atenet-dataplane=envoy)",
},

{
// No default: an unset level keeps the one an existing install has,
Expand Down
6 changes: 6 additions & 0 deletions cmd/ate-setup/internal/steps/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error {
if err := e.requireNoWorkerPods(ctx); err != nil {
return err
}
if err := e.requireTokenExchangeCA(ctx); err != nil {
return err
}
var microvmPrep *microVMPrepared
if microvm {
microvmPrep, err = e.prepareMicroVMDeps(ctx)
Expand Down Expand Up @@ -556,6 +559,9 @@ func (e *Env) DeployAtenet(ctx context.Context) error {
if err != nil {
return err
}
if err := e.requireTokenExchangeCA(ctx); err != nil {
return err
}
if err := e.EnsureCRDs(ctx); err != nil {
return err
}
Expand Down
63 changes: 63 additions & 0 deletions cmd/ate-setup/internal/steps/overlay.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,10 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, providers []config
if err != nil {
return nil, err
}
raw, err = patchAtenetEgressTokenExchangeCA(raw, e.Cfg.TokenExchangeCAConfigMap)
if err != nil {
return nil, err
}
raw = e.patchEnvoyDataplaneImage(raw, imageReference)
if e.Cfg.EnvoyConcurrency > 0 {
raw, err = patchAtenetEgressEnvoyConcurrency(raw, e.Cfg.EnvoyConcurrency)
Expand Down Expand Up @@ -389,6 +393,65 @@ func patchAgentgatewayEgressInject(raw []byte, providers []config.CredentialProv
return replaceManifestMarker(raw, "#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS", block)
}

const (
tokenExchangeCADir = "/run/token-exchange-ca"
tokenExchangeCAKey = "ca.crt"
)

// patchAtenetEgressTokenExchangeCA replaces the token exchange CA markers in
// the egress sidecar with --token-exchange-ca-file and a mount of configMap's
// ca.crt, or removes them when configMap is empty.
func patchAtenetEgressTokenExchangeCA(raw []byte, configMap string) ([]byte, error) {
var flag, mount, volume string
if configMap != "" {
flag = fmt.Sprintf("- --token-exchange-ca-file=%s/%s", tokenExchangeCADir, tokenExchangeCAKey)
mount = fmt.Sprintf(`- name: token-exchange-ca
mountPath: %s
readOnly: true`, tokenExchangeCADir)
volume = fmt.Sprintf(`- name: token-exchange-ca
configMap:
name: %q
items:
- key: %s
path: %s`, configMap, tokenExchangeCAKey, tokenExchangeCAKey)
}
for _, splice := range []struct{ marker, block string }{
{"#ATE_TOKEN_EXCHANGE_CA_FLAG", flag},
{"#ATE_TOKEN_EXCHANGE_CA_MOUNT", mount},
{"#ATE_TOKEN_EXCHANGE_CA_VOLUME", volume},
} {
var err error
if raw, err = replaceManifestMarker(raw, splice.marker, splice.block); err != nil {
return nil, err
}
}
return raw, nil
}

// requireTokenExchangeCA fails when --token-exchange-ca-configmap names a
// ConfigMap the install namespace lacks, or one without a ca.crt key, which
// would leave the egress gateway unable to start.
func (e *Env) requireTokenExchangeCA(ctx context.Context) error {
name := e.Cfg.TokenExchangeCAConfigMap
if name == "" {
return nil
}
cm, err := e.Kube.GetConfigMap(ctx, e.Namespace(), name)
if err != nil {
return err
}
if cm == nil {
return fmt.Errorf("token exchange CA ConfigMap %s/%s does not exist; create it with the CA certificates under key %s",
e.Namespace(), name, tokenExchangeCAKey)
}
_, inData := cm.Data[tokenExchangeCAKey]
_, inBinary := cm.BinaryData[tokenExchangeCAKey]
if !inData && !inBinary {
return fmt.Errorf("token exchange CA ConfigMap %s/%s has no %s key", e.Namespace(), name, tokenExchangeCAKey)
}
return nil
}

func replaceManifestMarker(raw []byte, marker, block string) ([]byte, error) {
var out []string
replaced := 0
Expand Down
99 changes: 99 additions & 0 deletions cmd/ate-setup/internal/steps/overlay_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1623,6 +1623,105 @@ func TestRenderAtenetEgressManifestEnvoyConcurrency(t *testing.T) {
})
}

func TestRenderAtenetEgressManifestTokenExchangeCA(t *testing.T) {
const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"}
render := func(t *testing.T, configMap string) (corev1.Container, []corev1.Volume) {
t.Helper()
e := &Env{
Cfg: &config.Config{Root: repoRoot(t), Router: config.RouterEnvoy, Images: src, TokenExchangeCAConfigMap: configMap},
resolver: images.NewPrebuilt(src, func(context.Context, string) (string, error) {
return digest, nil
}),
}
// No credential provider: the CA does not depend on one.
out, err := e.renderAtenetEgressManifest(t.Context(), nil)
if err != nil {
t.Fatalf("renderAtenetEgressManifest() error = %v", err)
}
for _, doc := range strings.Split(string(out), "\n---\n") {
var dep appsv1.Deployment
if err := yaml.Unmarshal([]byte(doc), &dep); err != nil || dep.Kind != "Deployment" || dep.Name != "atenet-egress" {
continue
}
for _, c := range dep.Spec.Template.Spec.Containers {
if c.Name == "ext-proc" {
return c, dep.Spec.Template.Spec.Volumes
}
}
t.Fatal("atenet-egress Deployment has no ext-proc container")
}
t.Fatal("rendered manifest has no atenet-egress Deployment")
return corev1.Container{}, nil
}
const flag = "--token-exchange-ca-file=/run/token-exchange-ca/ca.crt"

t.Run("unset", func(t *testing.T) {
extProc, volumes := render(t, "")
if slices.Contains(extProc.Args, flag) {
t.Errorf("ext-proc args contain %s with no ConfigMap set", flag)
}
if slices.ContainsFunc(extProc.VolumeMounts, func(m corev1.VolumeMount) bool { return m.Name == "token-exchange-ca" }) {
t.Error("ext-proc mounts token-exchange-ca with no ConfigMap set")
}
if slices.ContainsFunc(volumes, func(v corev1.Volume) bool { return v.Name == "token-exchange-ca" }) {
t.Error("atenet-egress has a token-exchange-ca volume with no ConfigMap set")
}
})

t.Run("set", func(t *testing.T) {
extProc, volumes := render(t, "corp-ca")
if !slices.Contains(extProc.Args, flag) {
t.Errorf("ext-proc args = %v, want %s", extProc.Args, flag)
}
wantMount := corev1.VolumeMount{Name: "token-exchange-ca", MountPath: "/run/token-exchange-ca", ReadOnly: true}
if !slices.ContainsFunc(extProc.VolumeMounts, func(m corev1.VolumeMount) bool { return reflect.DeepEqual(m, wantMount) }) {
t.Errorf("ext-proc volume mounts = %v, want %v", extProc.VolumeMounts, wantMount)
}
wantVolume := corev1.Volume{Name: "token-exchange-ca", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{Name: "corp-ca"},
Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}},
}}}
if !slices.ContainsFunc(volumes, func(v corev1.Volume) bool { return reflect.DeepEqual(v, wantVolume) }) {
t.Errorf("atenet-egress volumes = %v, want one equal to %v", volumes, wantVolume)
}
})

if _, err := patchAtenetEgressTokenExchangeCA([]byte("kind: ConfigMap\n"), "corp-ca"); err == nil {
t.Error("patchAtenetEgressTokenExchangeCA accepted a manifest without the markers")
}
}

func TestRequireTokenExchangeCA(t *testing.T) {
configMap := func(data, binaryData map[string]string) *corev1.ConfigMap {
cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: "corp-ca"}, Data: data}
for k, v := range binaryData {
cm.BinaryData = map[string][]byte{k: []byte(v)}
}
return cm
}
for _, tc := range []struct {
name string
configMap string
objects []runtime.Object
wantErr bool
}{
{name: "unset"},
{name: "present", configMap: "corp-ca", objects: []runtime.Object{configMap(map[string]string{"ca.crt": "pem"}, nil)}},
{name: "present as binary data", configMap: "corp-ca", objects: []runtime.Object{configMap(nil, map[string]string{"ca.crt": "der"})}},
{name: "missing", configMap: "corp-ca", wantErr: true},
{name: "without ca.crt", configMap: "corp-ca", objects: []runtime.Object{configMap(map[string]string{"tls.crt": "pem"}, nil)}, wantErr: true},
} {
t.Run(tc.name, func(t *testing.T) {
e := &Env{Cfg: &config.Config{Root: repoRoot(t), TokenExchangeCAConfigMap: tc.configMap}, Kube: fakeKube(t, tc.objects...)}
err := e.requireTokenExchangeCA(t.Context())
if (err != nil) != tc.wantErr {
t.Fatalf("requireTokenExchangeCA() = %v, want error %v", err, tc.wantErr)
}
})
}
}

// Without --fake-data-plane there is no fake-workersync rollout to wait for.
func TestWaitFakeWorkersyncWithoutTheFlagIsANoOp(t *testing.T) {
e := &Env{Cfg: &config.Config{Root: repoRoot(t)}, Kube: fakeKube(t)}
Expand Down
Loading
Loading