Router authn/z part 2: ate-setup: add --ingress-auth-mode

AGENTS.md

@@ -104,5 +104,6 @@ Keep this up to date when updating AGENTS.md.
 
 - **Workload Isolation**: The project uses `gVisor` (`runsc`) for sandboxing and security isolation of workloads on pods.
 - **JWTs and JWKs**: Signing (`internal/localjwtauthority`), verification (`cmd/ateapi/internal/oidcjwt`), and JWK set publishing (`internal/oidcdiscovery`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses.
+- **Ingress Authentication**: `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a SPIFFE certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids`; that list is the whole authorization policy, so any listed client may reach any Actor. The default, `deprecated-insecure`, checks a certificate only when a client presents one, and also serves plaintext listeners that accept any client. Install with it via `ate-setup --ingress-auth-mode=static-mtls` (`manifests/ate-install/components/router-static-mtls`); the allowlist lives in `manifests/ate-install/atenet-router.yaml`.
 
 For future plans for security, reference `docs/roadmap.md`.

cmd/ate-setup/internal/cmd/deploy.go

@@ -59,8 +59,8 @@ The bundled PostgreSQL StatefulSet is skipped when
 ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* variables
 select an external database.
 
-Shape the install with the global --atenet-dataplane, --cluster-size, and
---cordon-control-plane flags.`,
+Shape the install with the global --atenet-dataplane, --cluster-size,
+--cordon-control-plane flags, and --ingress-auth-mode flags.`,
 	// Args runs before the root command loads the configuration, so only the
 	// flag is readable here. Checking it keeps an unusable value from costing
 	// a credential fetch, which is the common case; a value arriving from the

cmd/ate-setup/internal/config/config.go

@@ -50,6 +50,11 @@ const (
 	// the podcertificate controller's API rate limits to match.
 	ClusterSizeSize0  = "size0"
 	ClusterSizeSize10 = "size10"
+
+	// Ingress router authentication modes; see atenet router's
+	// --ingress-auth-mode.
+	IngressAuthDeprecatedInsecure = "deprecated-insecure"
+	IngressAuthStaticMTLS         = "static-mtls"
 )
 
 // DefaultRolloutTimeout is the default wait timeout for workload rollouts.
@@ -187,6 +192,11 @@ type Config struct {
 	// ate.dev/workloadType=ate-postgres:NoSchedule for postgres alone.
 	CordonControlPlane bool
 
+	// IngressAuthMode is how the ingress router authenticates its clients
+	// (ATE_INGRESS_AUTH_MODE): IngressAuthDeprecatedInsecure or
+	// IngressAuthStaticMTLS.
+	IngressAuthMode string
+
 	// AdditionalEgressExtprocService is the optional NS/SVC:PORT external processor filter.
 	AdditionalEgressExtprocService string
 
@@ -422,6 +432,10 @@ func (c *Config) ScriptEnv() []string {
 	if c.CordonControlPlane {
 		merged["ATE_INSTALL_CORDON_CONTROL_PLANE"] = "true"
 	}
+	delete(merged, "ATE_INGRESS_AUTH_MODE")
+	if c.IngressAuthMode != "" && c.IngressAuthMode != IngressAuthDeprecatedInsecure {
+		merged["ATE_INGRESS_AUTH_MODE"] = c.IngressAuthMode
+	}
 	if c.AdditionalEgressExtprocService != "" {
 		merged["ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE"] = c.AdditionalEgressExtprocService
 	}

cmd/ate-setup/internal/config/config_test.go

@@ -55,6 +55,7 @@ func loadEnv(t *testing.T) {
 		"ATE_CREDENTIAL_PROVIDER",
 		"ATE_IMAGE_REPO",
 		"ATE_IMAGE_TAG",
+		"ATE_INGRESS_AUTH_MODE",
 		"ATE_INSTALL_CLUSTER_SIZE",
 		"ATE_INSTALL_CORDON_CONTROL_PLANE",
 		"ATE_INSTALL_KIND",
@@ -118,6 +119,9 @@ func TestLoadDefaults(t *testing.T) {
 	if cfg.CordonControlPlane {
 		t.Error("CordonControlPlane = true, want false")
 	}
+	if cfg.IngressAuthMode != IngressAuthDeprecatedInsecure {
+		t.Errorf("IngressAuthMode = %q, want %q", cfg.IngressAuthMode, IngressAuthDeprecatedInsecure)
+	}
 }
 
 func TestLoadClusterSize(t *testing.T) {
@@ -206,6 +210,39 @@ func TestLoadCordonControlPlane(t *testing.T) {
 	}
 }
 
+func TestLoadIngressAuthMode(t *testing.T) {
+	for _, tc := range []struct {
+		name       string
+		opts       Options
+		env        string
+		want       string
+		wantExport bool
+	}{
+		{name: "flag static-mtls", opts: Options{IngressAuthMode: IngressAuthStaticMTLS}, want: IngressAuthStaticMTLS, wantExport: true},
+		{name: "environment static-mtls", env: IngressAuthStaticMTLS, want: IngressAuthStaticMTLS, wantExport: true},
+		{name: "environment deprecated-insecure", env: IngressAuthDeprecatedInsecure, want: IngressAuthDeprecatedInsecure},
+		{name: "flag outranks environment", opts: Options{IngressAuthMode: IngressAuthDeprecatedInsecure}, env: IngressAuthStaticMTLS, want: IngressAuthDeprecatedInsecure},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			loadEnv(t)
+			if tc.env != "" {
+				t.Setenv("ATE_INGRESS_AUTH_MODE", tc.env)
+			}
+			cfg, err := Load(tc.opts)
+			if err != nil {
+				t.Fatalf("Load() error = %v", err)
+			}
+			if cfg.IngressAuthMode != tc.want {
+				t.Errorf("IngressAuthMode = %q, want %q", cfg.IngressAuthMode, tc.want)
+			}
+			got, exported := scriptEnvMap(t, cfg)["ATE_INGRESS_AUTH_MODE"]
+			if exported != tc.wantExport || (exported && got != tc.want) {
+				t.Errorf("ScriptEnv() ATE_INGRESS_AUTH_MODE = %q (exported %v), want %q (exported %v)", got, exported, tc.want, tc.wantExport)
+			}
+		})
+	}
+}
+
 func TestLoadDockerBuildFlags(t *testing.T) {
 	loadEnv(t)
 	t.Setenv("DOCKER_BUILD_FLAGS", " --cache-from type=gha  --cache-to type=gha,mode=max ")
@@ -624,6 +661,9 @@ func TestLoadRejectsInvalidValues(t *testing.T) {
 		{"kubernetes address without a port", Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc"}`}},
 		{"other provider without an address", Options{CredentialProvider: `{"name":"vault.example.com"}`}},
 		{"other provider address without a port", Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc"}`}},
+		{"ingress auth mode", Options{IngressAuthMode: "mtls"}},
+		{"old insecure ingress auth mode", Options{IngressAuthMode: "insecure"}},
+		{"static-mtls agentgateway", Options{Router: RouterAgentgateway, IngressAuthMode: IngressAuthStaticMTLS}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			if _, err := Load(tc.opts); err == nil {

cmd/ate-setup/internal/config/coverage_test.go

@@ -90,6 +90,8 @@ func valueFor(key string) string {
 		return RouterEnvoy
 	case "clusterSize":
 		return ClusterSizeSize0
+	case "ingressAuthMode":
+		return IngressAuthStaticMTLS
 	case "ateapi.postgres.cloudsql.ipType":
 		return CloudSQLIPTypePSC
 	case "atenet.egress.additionalExtprocService":

cmd/ate-setup/internal/config/load.go

@@ -245,6 +245,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro
 		PodcertWorkersPerSigner:        podcertWorkers,
 		ClusterSize:                    r.String("clusterSize"),
 		CordonControlPlane:             r.Bool("cordonControlPlane"),
+		IngressAuthMode:                r.String("ingressAuthMode"),
 		AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"),
 		CredentialProviderJSON:         r.String("atenet.egress.credentialProvider"),
 		AnthropicAPIKey:                r.String("demo.anthropicAPIKey"),
@@ -361,6 +362,18 @@ func validateResolved(cfg *Config, r *Resolved) error {
 		return &InvalidError{Value: size, Want: ClusterSizeSize0 + " or " + ClusterSizeSize10}
 	}
 
+	authMode, _ := r.Value("ingressAuthMode")
+	switch cfg.IngressAuthMode {
+	case IngressAuthDeprecatedInsecure:
+	case IngressAuthStaticMTLS:
+		if cfg.Router != RouterEnvoy {
+			return &ConflictError{A: authMode, B: dataplane,
+				Why: IngressAuthStaticMTLS + " requires dataplane " + RouterEnvoy}
+		}
+	default:
+		return &InvalidError{Value: authMode, Want: IngressAuthDeprecatedInsecure + " or " + IngressAuthStaticMTLS}
+	}
+
 	switch cfg.ActorJWTAlgorithm {
 	case "ES256", "RS256":
 	default:

cmd/ate-setup/internal/config/options_test.go

@@ -38,6 +38,7 @@ type Options struct {
 	PodcertWorkersPerSigner        int
 	ClusterSize                    string
 	CordonControlPlane             bool
+	IngressAuthMode                string
 	AdditionalEgressExtprocService string
 	CredentialProvider             string
 	OtlpEndpoint                   string
@@ -86,6 +87,7 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) {
 		{"atenet-dataplane", opts.Router},
 		{"rollout-timeout", opts.RolloutTimeout},
 		{"cluster-size", opts.ClusterSize},
+		{"ingress-auth-mode", opts.IngressAuthMode},
 		{"experimental-additional-egress-extproc-service", opts.AdditionalEgressExtprocService},
 		{"credential-provider", opts.CredentialProvider},
 		{"otlp-endpoint", opts.OtlpEndpoint},

cmd/ate-setup/internal/config/setting.go

@@ -261,6 +261,14 @@ var Registry = []Setting{
 			"and tainted ate.dev/workloadType=ate-control-plane:NoSchedule, and a one-node pool " +
 			"labeled and tainted ate.dev/workloadType=ate-postgres:NoSchedule for postgres alone",
 	},
+	{
+		Key: "ingressAuthMode", Env: "ATE_INGRESS_AUTH_MODE", Flag: "ingress-auth-mode", Kind: KindString,
+		Default: IngressAuthDeprecatedInsecure,
+		Usage: "How the ingress router authenticates clients: deprecated-insecure or static-mtls. Both check a " +
+			"client certificate on the router's TLS ports against the podidentity CA and the SPIFFE IDs listed in " +
+			"manifests/ate-install/atenet-router.yaml; deprecated-insecure also lets through TLS clients without one " +
+			"and serves plaintext ports, while static-mtls requires the certificate and --atenet-dataplane=envoy",
+	},
 	{
 		Key: "otlpEndpoint", Env: "ATE_OTLP_ENDPOINT", Flag: "otlp-endpoint", Kind: KindString,
 		Usage: "Send control plane telemetry to this OTLP collector instead of the cluster default",

cmd/ate-setup/internal/steps/overlay.go

@@ -41,6 +41,26 @@ const (
 // apply under --cordon-control-plane.
 const cordonControlPlaneComponent = installDir + "/components/cordon-control-plane"
 
+// routerStaticMTLSComponent is the kustomize component that switches the
+// ingress router to --ingress-auth-mode=static-mtls, layered over every
+// control plane apply under --ingress-auth-mode=static-mtls.
+const routerStaticMTLSComponent = installDir + "/components/router-static-mtls"
+
+// components lists the kustomize components every control plane apply is
+// composed with. Each patch targets its workloads by name, and kustomize leaves
+// a stream alone when nothing in it matches, so composing a manifest that
+// carries none of them is harmless.
+func (e *Env) components() []string {
+	var components []string
+	if e.Cfg.CordonControlPlane {
+		components = append(components, e.Cfg.Path(cordonControlPlaneComponent))
+	}
+	if e.Cfg.IngressAuthMode == config.IngressAuthStaticMTLS {
+		components = append(components, e.Cfg.Path(routerStaticMTLSComponent))
+	}
+	return components
+}
+
 // SystemOverlay picks the kustomization for a full control plane install.
 //
 // The choice is a product of two switches: kind vs GKE, and the atenet router
@@ -64,15 +84,12 @@ func SystemOverlay(cfg *config.Config) string {
 }
 
 // render emits the manifests at path, an absolute manifest file or
-// kustomization directory, before image resolution. Under
-// --cordon-control-plane it composes path with the cordon-control-plane
-// component, so the same node pinning reaches every control plane workload
-// whichever apply path delivers it. The component's patch has a name-regex
-// target and kustomize leaves a stream alone when nothing in it matches, so
-// wrapping a manifest that carries none of those workloads is harmless.
+// kustomization directory, before image resolution. It composes path with the
+// selected components (see components), so the same patches reach every
+// control plane workload whichever apply path delivers it.
 func (e *Env) render(path string) ([]byte, error) {
-	if e.Cfg.CordonControlPlane {
-		return kustomize.Compose(path, e.Cfg.Path(cordonControlPlaneComponent))
+	if components := e.components(); len(components) > 0 {
+		return kustomize.Compose(path, components...)
 	}
 	info, err := os.Stat(path)
 	if err != nil {
@@ -90,8 +107,8 @@ func (e *Env) render(path string) ([]byte, error) {
 
 // renderBytes is render for a manifest already held in memory.
 func (e *Env) renderBytes(manifest []byte) ([]byte, error) {
-	if e.Cfg.CordonControlPlane {
-		return kustomize.ComposeBytes(manifest, e.Cfg.Path(cordonControlPlaneComponent))
+	if components := e.components(); len(components) > 0 {
+		return kustomize.ComposeBytes(manifest, components...)
 	}
 	return manifest, nil
 }

cmd/ate-setup/internal/steps/overlay_test.go

@@ -542,6 +542,100 @@ func TestRenderWithoutCordonLeavesManifestsAlone(t *testing.T) {
 	}
 }
 
+// Under --ingress-auth-mode=static-mtls both paths that deliver the router
+// -- the system bundle and the lone router redeploy -- have to carry the
+// static-mtls flags, alone or alongside the cordon component. Whether the
+// router accepts those flags is pinned on the router side, by
+// TestRouterIngressAuthManifestsAreValid.
+func TestRenderRouterStaticMTLS(t *testing.T) {
+	root := repoRoot(t)
+	for _, tc := range []struct {
+		name   string
+		cfg    config.Config
+		path   func(e *Env) string
+		cordon bool
+	}{
+		{
+			name: "base bundle",
+			cfg:  config.Config{Router: config.RouterEnvoy},
+			path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) },
+		},
+		{
+			name: "kind bundle",
+			cfg:  config.Config{Router: config.RouterEnvoy, Kind: true},
+			path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) },
+		},
+		{
+			name: "router file",
+			cfg:  config.Config{Router: config.RouterEnvoy},
+			path: func(e *Env) string { return e.Cfg.Manifest("atenet-router.yaml") },
+		},
+		{
+			name:   "kind bundle with cordon",
+			cfg:    config.Config{Router: config.RouterEnvoy, Kind: true},
+			path:   func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) },
+			cordon: true,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := tc.cfg
+			cfg.Root = root
+			cfg.IngressAuthMode = config.IngressAuthStaticMTLS
+			cfg.CordonControlPlane = tc.cordon
+			e := &Env{Cfg: &cfg}
+
+			rendered, err := e.render(tc.path(e))
+			if err != nil {
+				t.Fatalf("render: %v", err)
+			}
+			args := routerArgs(t, rendered)
+			if !slices.Contains(args, "--ingress-auth-mode=static-mtls") {
+				t.Errorf("atenet-router args %v do not select static-mtls", args)
+			}
+			if tc.cordon {
+				if pinned, _ := pinnedWorkloads(t, rendered); pinned["atenet-router"] != "ate-control-plane" {
+					t.Errorf("composing both components dropped the cordon pinning (pinned: %v)", pinned)
+				}
+			}
+		})
+	}
+
+	t.Run("deprecated-insecure leaves the router alone", func(t *testing.T) {
+		e := &Env{Cfg: &config.Config{Root: root, Router: config.RouterEnvoy, IngressAuthMode: config.IngressAuthDeprecatedInsecure}}
+		rendered, err := e.render(e.Cfg.Manifest("atenet-router.yaml"))
+		if err != nil {
+			t.Fatalf("render: %v", err)
+		}
+		for _, arg := range routerArgs(t, rendered) {
+			if strings.HasPrefix(arg, "--ingress-auth-mode") {
+				t.Errorf("deprecated-insecure render sets %s", arg)
+			}
+		}
+	})
+}
+
+// routerArgs returns the atenet-router container's args from a rendered
+// manifest.
+func routerArgs(t *testing.T, manifest []byte) []string {
+	t.Helper()
+	for _, doc := range strings.Split(string(manifest), "\n---\n") {
+		var deployment appsv1.Deployment
+		if err := yaml.Unmarshal([]byte(doc), &deployment); err != nil {
+			t.Fatalf("rendered document is not valid YAML: %v", err)
+		}
+		if deployment.Kind != "Deployment" || deployment.Name != "atenet-router" {
+			continue
+		}
+		for _, c := range deployment.Spec.Template.Spec.Containers {
+			if c.Name == "atenet-router" {
+				return c.Args
+			}
+		}
+	}
+	t.Fatal("rendered manifest has no atenet-router container")
+	return nil
+}
+
 // The agentgateway egress overlay mounts the CA pool Secret
 // EnsureEgressMITMCAPoolSecret generates; without it atenet-egress waits on a
 // Secret nobody creates.

cmd/atenet/internal/router/README.md

@@ -194,6 +194,45 @@ independently, not because they need separate binaries.
 `--atenet-dataplane` selects the dataplane for both Deployments. Each gateway has
 its own static configuration because ingress and egress scale independently.
 
+## ingress client authentication
+
+`--ingress-auth-mode` selects who may send traffic to actors through the
+ingress listeners. In both modes the TLS listeners (`--port-https`,
+`--port-connect-tls`) ask clients for a certificate and reject one that does
+not chain to `--ingress-client-ca-file` or whose URI SAN does not exactly
+match one of `--ingress-allowed-spiffe-ids`. The modes differ in what happens
+to a client that presents none, and in whether the plaintext listeners run:
+
+| `--ingress-auth-mode` | plaintext (`--port-http`, `--port-connect`) | TLS, no client certificate | TLS, client certificate |
+| --- | --- | --- | --- |
+| `deprecated-insecure` (default) | anyone | let through | must pass the allowlist |
+| `static-mtls` | must be disabled | rejected | must pass the allowlist |
+
+`deprecated-insecure` lets clients move to mTLS one at a time: a client can
+start presenting its certificate, and learn whether it is accepted, while
+everyone else carries on without one. It authenticates nobody who does not opt
+in, and goes away once every client has.
+
+Envoy enforces client authentication in the TLS handshake (a validation
+context with URI SAN matchers on every downstream TLS context, plus
+`require_client_certificate` in `static-mtls`), so a refused client never
+reaches ext_proc and cannot resume an actor. The CA bundle reaches Envoy over SDS with a watched directory, like the
+serving certificate, so a rotated ClusterTrustBundle is picked up without a
+restart. Envoy reads the file, so it must be mounted at the same path in the
+envoy container.
+
+The router refuses to start with a TLS listener enabled but no client CA or
+allowlist. In `static-mtls` it also refuses to start with a plaintext listener
+enabled (`--port-http` or `--port-connect` above 0), without a TLS listener,
+or with `--atenet-dataplane=agentgateway`, whose configuration is static;
+`deprecated-insecure` leaves agentgateway's statically configured listeners
+alone. The allowlist is the whole authorization policy: any listed client may
+reach any actor. The base manifest (`manifests/ate-install/atenet-router.yaml`)
+sets the client CA and lists the allowed SPIFFE IDs;
+`ate-setup --ingress-auth-mode=static-mtls` layers on
+`manifests/ate-install/components/router-static-mtls`, which switches the mode
+and disables the plaintext listeners.
+
 ## status page
 
 Serve a `/statusz` page on port 8080.