feat(MeshService): support mTLS (#10403)

Add support for mTLS by verifying that the destination's cert comes from one of the identities (`kuma.io/service` values) matched by this MeshService's selector.

Signed-off-by: Mike Beaumont <mjboamail@gmail.com>

pkg/plugins/policies/core/xds/meshroute/clusters.go

@@ -6,6 +6,7 @@ import (
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
+	"github.com/kumahq/kuma/pkg/util/maps"
 	xds_context "github.com/kumahq/kuma/pkg/xds/context"
 	envoy_common "github.com/kumahq/kuma/pkg/xds/envoy"
 	envoy_clusters "github.com/kumahq/kuma/pkg/xds/envoy/clusters"
@@ -75,9 +76,16 @@ func GenerateClusters(
 						}
 					}
 				} else {
-					edsClusterBuilder.Configure(envoy_clusters.ClientSideMTLS(
-						proxy.SecretsTracker,
-						meshCtx.Resource, serviceName, tlsReady, clusterTags))
+					if msName := service.MeshServiceName(); len(msName) > 0 {
+						identities := meshCtx.MeshServiceIdentity[msName].Identities
+						edsClusterBuilder.Configure(envoy_clusters.ClientSideMultiIdentitiesMTLS(
+							proxy.SecretsTracker,
+							meshCtx.Resource, tlsReady, clusterTags, maps.SortedKeys(identities)))
+					} else {
+						edsClusterBuilder.Configure(envoy_clusters.ClientSideMTLS(
+							proxy.SecretsTracker,
+							meshCtx.Resource, serviceName, tlsReady, clusterTags))
+					}
 				}
 			}
 

pkg/plugins/policies/core/xds/meshroute/listeners.go

@@ -74,13 +74,13 @@ func CollectServices(
 			if outbound.GetAddress() == proxy.Dataplane.Spec.GetNetworking().GetAddress() {
 				continue
 			}
-			ms, ok := meshCtx.MeshServiceByName[outbound.BackendRef.Name]
+			ms, ok := meshCtx.MeshServiceIdentity[outbound.BackendRef.Name]
 			if !ok {
 				// we want to ignore service which is not found. Logging might be excessive here.
 				// We don't have other mechanism to bubble up warnings yet.
 				continue
 			}
-			port, ok := ms.FindPort(outbound.BackendRef.Port)
+			port, ok := ms.Resource.FindPort(outbound.BackendRef.Port)
 			if !ok {
 				continue
 			}
@@ -91,11 +91,11 @@ func CollectServices(
 			dests = append(dests, DestinationService{
 				Outbound:    oface,
 				Protocol:    protocol,
-				ServiceName: ms.DestinationName(outbound.BackendRef.Port),
+				ServiceName: ms.Resource.DestinationName(outbound.BackendRef.Port),
 				BackendRef: common_api.BackendRef{
 					TargetRef: common_api.TargetRef{
 						Kind: common_api.MeshService,
-						Name: ms.GetMeta().GetName(),
+						Name: ms.Resource.GetMeta().GetName(),
 					},
 					Port: &port.Port,
 				},
@@ -140,16 +140,18 @@ func makeSplit(
 		if pointer.DerefOr(ref.Weight, 1) == 0 {
 			continue
 		}
+		var meshServiceName string
 		if ref.Port != nil { // in this case, reference real MeshService instead of kuma.io/service tag
-			ms, ok := meshCtx.MeshServiceByName[ref.Name]
+			ms, ok := meshCtx.MeshServiceIdentity[ref.Name]
 			if !ok {
 				continue
 			}
-			port, ok := ms.FindPort(*ref.Port)
+			meshServiceName = ms.Resource.GetMeta().GetName()
+			port, ok := ms.Resource.FindPort(*ref.Port)
 			if !ok {
 				continue
 			}
-			service = ms.DestinationName(*ref.Port)
+			service = ms.Resource.DestinationName(*ref.Port)
 			protocol = port.Protocol // todo(jakubdyszkiewicz): do we need to default to TCP or will this be done by MeshService defaulter?
 		} else {
 			service = ref.Name
@@ -204,7 +206,11 @@ func makeSplit(
 			clusterBuilder.WithMesh(mesh)
 		}
 
-		servicesAcc.Add(clusterBuilder.Build())
+		if len(meshServiceName) > 0 {
+			servicesAcc.AddMeshService(meshServiceName, clusterBuilder.Build())
+		} else {
+			servicesAcc.Add(clusterBuilder.Build())
+		}
 	}
 
 	return split

pkg/plugins/policies/meshhttproute/plugin/v1alpha1/plugin_test.go

@@ -19,6 +19,7 @@ import (
 	core_plugins "github.com/kumahq/kuma/pkg/core/plugins"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
 	meshservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1"
+	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
 	"github.com/kumahq/kuma/pkg/core/secrets/cipher"
 	secret_manager "github.com/kumahq/kuma/pkg/core/secrets/manager"
 	secret_store "github.com/kumahq/kuma/pkg/core/secrets/store"
@@ -39,6 +40,7 @@ import (
 	util_proto "github.com/kumahq/kuma/pkg/util/proto"
 	"github.com/kumahq/kuma/pkg/xds/cache/cla"
 	xds_context "github.com/kumahq/kuma/pkg/xds/context"
+	envoy "github.com/kumahq/kuma/pkg/xds/envoy"
 )
 
 func getResource(resourceSet *core_xds.ResourceSet, typ envoy_resource.Type) []byte {
@@ -119,11 +121,16 @@ var _ = Describe("MeshHTTPRoute", func() {
 		}()),
 		Entry("default-meshservice", func() outboundsTestCase {
 			outboundTargets := xds_builders.EndpointMap().
-				AddEndpoint("backend_svc_8084", xds_builders.Endpoint().
+				AddEndpoint("backend_svc_80", xds_builders.Endpoint().
 					WithTarget("192.168.0.4").
 					WithPort(8084).
 					WithWeight(1).
-					WithTags(mesh_proto.ServiceTag, "backend", mesh_proto.ProtocolTag, core_mesh.ProtocolHTTP, "region", "us"))
+					WithTags(mesh_proto.ServiceTag, "backend", mesh_proto.ProtocolTag, core_mesh.ProtocolHTTP, "app", "backend")).
+				AddEndpoint("backend_svc_80", xds_builders.Endpoint().
+					WithTarget("192.168.0.5").
+					WithPort(8084).
+					WithWeight(1).
+					WithTags(mesh_proto.ServiceTag, "other-backend", mesh_proto.ProtocolTag, core_mesh.ProtocolHTTP, "app", "backend"))
 			meshSvc := meshservice_api.MeshServiceResource{
 				Meta: &test_model.ResourceMeta{Name: "backend", Mesh: "default"},
 				Spec: &meshservice_api.MeshService{
@@ -146,11 +153,13 @@ var _ = Describe("MeshHTTPRoute", func() {
 			}
 			return outboundsTestCase{
 				xdsContext: *xds_builders.Context().
+					WithMesh(builders.Mesh().WithBuiltinMTLSBackend("builtin").WithEnabledMTLSBackend("builtin")).
 					WithEndpointMap(outboundTargets).
 					WithResources(resources).
 					AddServiceProtocol("backend_svc_80", core_mesh.ProtocolHTTP).
 					Build(),
 				proxy: xds_builders.Proxy().
+					WithSecretsTracker(envoy.NewSecretsTracker(core_model.DefaultMesh, nil)).
 					WithDataplane(samples.DataplaneWebBuilder().
 						AddOutbound(builders.Outbound().
 							WithAddress("10.0.0.1").

pkg/plugins/policies/meshhttproute/plugin/v1alpha1/testdata/default-meshservice.clusters.golden.yaml

@@ -7,6 +7,33 @@ resources:
         ads: {}
         resourceApiVersion: V3
     name: backend_svc_80
+    transportSocket:
+      name: envoy.transport_sockets.tls
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+        commonTlsContext:
+          alpnProtocols:
+          - kuma
+          combinedValidationContext:
+            defaultValidationContext:
+              matchTypedSubjectAltNames:
+              - matcher:
+                  exact: spiffe://default/backend
+                sanType: URI
+              - matcher:
+                  exact: spiffe://default/other-backend
+                sanType: URI
+            validationContextSdsSecretConfig:
+              name: mesh_ca:secret:default
+              sdsConfig:
+                ads: {}
+                resourceApiVersion: V3
+          tlsCertificateSdsSecretConfigs:
+          - name: identity_cert:secret:default
+            sdsConfig:
+              ads: {}
+              resourceApiVersion: V3
+        sni: backend{mesh=default}
     type: EDS
     typedExtensionProtocolOptions:
       envoy.extensions.upstreams.http.v3.HttpProtocolOptions:

pkg/plugins/policies/meshhttproute/plugin/v1alpha1/testdata/default-meshservice.endpoints.golden.yaml

@@ -3,3 +3,33 @@ resources:
   resource:
     '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
     clusterName: backend_svc_80
+    endpoints:
+    - lbEndpoints:
+      - endpoint:
+          address:
+            socketAddress:
+              address: 192.168.0.4
+              portValue: 8084
+        loadBalancingWeight: 1
+        metadata:
+          filterMetadata:
+            envoy.lb:
+              app: backend
+              kuma.io/protocol: http
+            envoy.transport_socket_match:
+              app: backend
+              kuma.io/protocol: http
+      - endpoint:
+          address:
+            socketAddress:
+              address: 192.168.0.5
+              portValue: 8084
+        loadBalancingWeight: 1
+        metadata:
+          filterMetadata:
+            envoy.lb:
+              app: backend
+              kuma.io/protocol: http
+            envoy.transport_socket_match:
+              app: backend
+              kuma.io/protocol: http

pkg/test/xds/builders/context_builder.go

@@ -2,12 +2,12 @@ package builders
 
 import (
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
-	meshservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1"
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
 	"github.com/kumahq/kuma/pkg/test/resources/builders"
 	"github.com/kumahq/kuma/pkg/test/resources/samples"
 	"github.com/kumahq/kuma/pkg/test/xds"
 	xds_context "github.com/kumahq/kuma/pkg/xds/context"
+	"github.com/kumahq/kuma/pkg/xds/topology"
 )
 
 type ContextBuilder struct {
@@ -21,7 +21,7 @@ func Context() *ContextBuilder {
 				Resource:            samples.MeshDefault(),
 				EndpointMap:         map[core_xds.ServiceName][]core_xds.Endpoint{},
 				ServicesInformation: map[string]*xds_context.ServiceInformation{},
-				MeshServiceByName:   map[string]*meshservice_api.MeshServiceResource{},
+				MeshServiceIdentity: map[string]topology.MeshServiceIdentity{},
 			},
 			ControlPlane: &xds_context.ControlPlaneContext{
 				CLACache: &xds.DummyCLACache{OutboundTargets: map[core_xds.ServiceName][]core_xds.Endpoint{}},
@@ -33,9 +33,9 @@ func Context() *ContextBuilder {
 }
 
 func (mc *ContextBuilder) Build() *xds_context.Context {
-	for _, ms := range mc.res.Mesh.Resources.MeshServices().Items {
-		mc.res.Mesh.MeshServiceByName[ms.GetMeta().GetName()] = ms
-	}
+	mc.res.Mesh.MeshServiceIdentity = topology.BuildMeshServiceIdentityMap(
+		mc.res.Mesh.Resources.MeshServices().Items, mc.res.Mesh.EndpointMap,
+	)
 	return mc.res
 }
 

pkg/xds/context/context.go

@@ -7,10 +7,10 @@ import (
 	"github.com/kumahq/kuma/pkg/core"
 	"github.com/kumahq/kuma/pkg/core/datasource"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
-	meshservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1"
 	"github.com/kumahq/kuma/pkg/core/xds"
 	"github.com/kumahq/kuma/pkg/xds/envoy"
 	"github.com/kumahq/kuma/pkg/xds/secrets"
+	"github.com/kumahq/kuma/pkg/xds/topology"
 )
 
 type Context struct {
@@ -63,7 +63,7 @@ type MeshContext struct {
 	Resource                    *core_mesh.MeshResource
 	Resources                   Resources
 	DataplanesByName            map[string]*core_mesh.DataplaneResource
-	MeshServiceByName           map[string]*meshservice_api.MeshServiceResource
+	MeshServiceIdentity         map[string]topology.MeshServiceIdentity
 	EndpointMap                 xds.EndpointMap
 	ExternalServicesEndpointMap xds.EndpointMap
 	CrossMeshEndpoints          map[xds.MeshName]xds.EndpointMap

pkg/xds/context/mesh_context_builder.go

@@ -15,7 +15,6 @@ import (
 	"github.com/kumahq/kuma/pkg/core/datasource"
 	"github.com/kumahq/kuma/pkg/core/dns/lookup"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
-	"github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1"
 	"github.com/kumahq/kuma/pkg/core/resources/apis/system"
 	"github.com/kumahq/kuma/pkg/core/resources/manager"
 	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
@@ -26,6 +25,7 @@ import (
 	"github.com/kumahq/kuma/pkg/log"
 	"github.com/kumahq/kuma/pkg/util/maps"
 	util_protocol "github.com/kumahq/kuma/pkg/util/protocol"
+	"github.com/kumahq/kuma/pkg/xds/topology"
 	xds_topology "github.com/kumahq/kuma/pkg/xds/topology"
 )
 
@@ -161,10 +161,6 @@ func (m *meshContextBuilder) BuildIfChanged(ctx context.Context, meshName string
 		dataplanesByName[dp.Meta.GetName()] = dp
 	}
 	meshServices := resources.MeshServices().Items
-	meshServicesByName := make(map[string]*v1alpha1.MeshServiceResource, len(dataplanes))
-	for _, ms := range meshServices {
-		meshServicesByName[ms.Meta.GetName()] = ms
-	}
 
 	var domains []xds.VIPDomains
 	var outbounds []*mesh_proto.Dataplane_Networking_Outbound
@@ -204,12 +200,14 @@ func (m *meshContextBuilder) BuildIfChanged(ctx context.Context, meshName string
 		)
 	}
 
+	meshServicesIdentity := topology.BuildMeshServiceIdentityMap(meshServices, endpointMap)
+
 	return &MeshContext{
 		Hash:                        newHash,
 		Resource:                    mesh,
 		Resources:                   resources,
 		DataplanesByName:            dataplanesByName,
-		MeshServiceByName:           meshServicesByName,
+		MeshServiceIdentity:         meshServicesIdentity,
 		EndpointMap:                 endpointMap,
 		ExternalServicesEndpointMap: esEndpointMap,
 		CrossMeshEndpoints:          crossMeshEndpointMap,

pkg/xds/envoy/clusters/configurers.go

@@ -36,6 +36,20 @@ func ClientSideMTLS(tracker core_xds.SecretsTracker, mesh *core_mesh.MeshResourc
 	})
 }
 
+func ClientSideMultiIdentitiesMTLS(tracker core_xds.SecretsTracker, mesh *core_mesh.MeshResource, upstreamTLSReady bool, tags []envoy_tags.Tags, identities []string) ClusterBuilderOpt {
+	return ClusterBuilderOptFunc(func(builder *ClusterBuilder) {
+		builder.AddConfigurer(&v3.ClientSideMTLSConfigurer{
+			SecretsTracker:   tracker,
+			UpstreamMesh:     mesh,
+			UpstreamService:  "*",
+			LocalMesh:        mesh,
+			Tags:             tags,
+			UpstreamTLSReady: upstreamTLSReady,
+			VerifyIdentities: identities,
+		})
+	})
+}
+
 func CrossMeshClientSideMTLS(tracker core_xds.SecretsTracker, localMesh *core_mesh.MeshResource, upstreamMesh *core_mesh.MeshResource, upstreamService string, upstreamTLSReady bool, tags []envoy_tags.Tags) ClusterBuilderOpt {
 	return ClusterBuilderOptFunc(func(builder *ClusterBuilder) {
 		builder.AddConfigurer(&v3.ClientSideMTLSConfigurer{

pkg/xds/envoy/clusters/v3/client_side_mtls_configurer.go

@@ -22,6 +22,7 @@ type ClientSideMTLSConfigurer struct {
 	LocalMesh        *core_mesh.MeshResource
 	Tags             []tags.Tags
 	UpstreamTLSReady bool
+	VerifyIdentities []string
 }
 
 var _ ClusterConfigurer = &ClientSideTLSConfigurer{}
@@ -79,7 +80,11 @@ func (c *ClientSideMTLSConfigurer) createTransportSocket(sni string) (*envoy_cor
 	ca := c.SecretsTracker.RequestCa(c.UpstreamMesh.GetMeta().GetName())
 	identity := c.SecretsTracker.RequestIdentityCert()
 
-	tlsContext, err := envoy_tls.CreateUpstreamTlsContext(identity, ca, c.UpstreamService, sni)
+	var verifyIdentities []string
+	if c.VerifyIdentities != nil {
+		verifyIdentities = c.VerifyIdentities
+	}
+	tlsContext, err := envoy_tls.CreateUpstreamTlsContext(identity, ca, c.UpstreamService, sni, verifyIdentities)
 	if err != nil {
 		return nil, err
 	}

pkg/xds/envoy/tls/v3/tls.go

@@ -35,12 +35,22 @@ func CreateDownstreamTlsContext(downstreamMesh core_xds.CaRequest, mesh core_xds
 // There is no way to correlate incoming request to "web" or "web-api" with outgoing request to "backend" to expose only one URI SAN.
 //
 // Pass "*" for upstreamService to validate that upstream service is a service that is part of the mesh (but not specific one)
-func CreateUpstreamTlsContext(mesh core_xds.IdentityCertRequest, upstreamMesh core_xds.CaRequest, upstreamService string, sni string) (*envoy_tls.UpstreamTlsContext, error) {
+func CreateUpstreamTlsContext(mesh core_xds.IdentityCertRequest, upstreamMesh core_xds.CaRequest, upstreamService string, sni string, verifyIdentities []string) (*envoy_tls.UpstreamTlsContext, error) {
 	var validationSANMatchers []*envoy_tls.SubjectAltNameMatcher
 	meshNames := upstreamMesh.MeshName()
 	for _, meshName := range meshNames {
 		if upstreamService == "*" {
-			validationSANMatchers = append(validationSANMatchers, MeshSpiffeIDPrefixMatcher(meshName))
+			if len(verifyIdentities) == 0 {
+				validationSANMatchers = append(validationSANMatchers, MeshSpiffeIDPrefixMatcher(meshName))
+			}
+			for _, identity := range verifyIdentities {
+				stringMatcher := ServiceSpiffeIDMatcher(meshName, identity)
+				matcher := &envoy_tls.SubjectAltNameMatcher{
+					SanType: envoy_tls.SubjectAltNameMatcher_URI,
+					Matcher: stringMatcher,
+				}
+				validationSANMatchers = append(validationSANMatchers, matcher)
+			}
 		} else {
 			stringMatcher := ServiceSpiffeIDMatcher(meshName, upstreamService)
 			matcher := &envoy_tls.SubjectAltNameMatcher{

pkg/xds/envoy/tls/v3/tls_test.go

@@ -114,6 +114,7 @@ var _ = Describe("CreateUpstreamTlsContext()", func() {
 					&caRequest{mesh: mesh},
 					given.upstreamService,
 					"",
+					nil,
 				)
 				// then
 				Expect(err).ToNot(HaveOccurred())