feat(crypto): control plane TLS support for Rate Limit Service phase I

internal/crypto/certgen.go

@@ -70,11 +70,13 @@ const (
 // Certificates contains a set of Certificates as []byte each holding
 // the CA Cert along with Envoy Gateway & Envoy certificates.
 type Certificates struct {
-	CACertificate           []byte
-	EnvoyGatewayCertificate []byte
-	EnvoyGatewayPrivateKey  []byte
-	EnvoyCertificate        []byte
-	EnvoyPrivateKey         []byte
+	CACertificate             []byte
+	EnvoyGatewayCertificate   []byte
+	EnvoyGatewayPrivateKey    []byte
+	EnvoyCertificate          []byte
+	EnvoyPrivateKey           []byte
+	EnvoyRateLimitCertificate []byte
+	EnvoyRateLimitPrivateKey  []byte
 }
 
 // certificateRequest defines a certificate request.
@@ -138,12 +140,27 @@ func GenerateCerts(cfg *config.Server) (*Certificates, error) {
 			return nil, err
 		}
 
+		envoyRateLimitCertReq := &certificateRequest{
+			caCertPEM:  caCertPEM,
+			caKeyPEM:   caKeyPEM,
+			expiry:     expiry,
+			commonName: DefaultEnvoyDNSPrefix,
+			altNames:   envoyDNSNames,
+		}
+
+		envoyRateLimitCert, envoyRateLimitKey, err := newCert(envoyRateLimitCertReq)
+		if err != nil {
+			return nil, err
+		}
+
 		return &Certificates{
-			CACertificate:           caCertPEM,
-			EnvoyGatewayCertificate: egCert,
-			EnvoyGatewayPrivateKey:  egKey,
-			EnvoyCertificate:        envoyCert,
-			EnvoyPrivateKey:         envoyKey,
+			CACertificate:             caCertPEM,
+			EnvoyGatewayCertificate:   egCert,
+			EnvoyGatewayPrivateKey:    egKey,
+			EnvoyCertificate:          envoyCert,
+			EnvoyPrivateKey:           envoyKey,
+			EnvoyRateLimitCertificate: envoyRateLimitCert,
+			EnvoyRateLimitPrivateKey:  envoyRateLimitKey,
 		}, nil
 	default:
 		// Envoy Gateway, e.g. self-signed CA, is the only supported certificate provider.

internal/crypto/certgen_test.go

@@ -89,6 +89,17 @@ func TestGeneratedValidKubeCerts(t *testing.T) {
 	envoyCert, _, err := newCert(envoyCertReq)
 	require.NoErrorf(t, err, "Failed to generate Envoy cert")
 
+	envoyRateLimitCertReq := &certificateRequest{
+		caCertPEM:  caCert,
+		caKeyPEM:   caKey,
+		expiry:     expiry,
+		commonName: "envoy",
+		altNames:   kubeServiceNames("envoy", "envoy-gateway-system", "cluster.local"),
+	}
+
+	envoyRateLimitCert, _, err := newCert(envoyRateLimitCertReq)
+	require.NoErrorf(t, err, "Failed to generate Envoy Rate Limit Client cert")
+
 	tests := []struct {
 		name    string
 		cert    []byte
@@ -104,6 +115,11 @@ func TestGeneratedValidKubeCerts(t *testing.T) {
 			cert:    envoyCert,
 			dnsName: "envoy",
 		},
+		{
+			name:    "envoy rate limit client cert",
+			cert:    envoyRateLimitCert,
+			dnsName: "envoy",
+		},
 	}
 
 	for i := range tests {

internal/infrastructure/kubernetes/ratelimit/resource.go

@@ -29,11 +29,11 @@ const (
 	// RedisTLSClientCertEnvVar is the redis tls client cert.
 	RedisTLSClientCertEnvVar = "REDIS_TLS_CLIENT_CERT"
 	// RedisTLSClientCertFilename is the reds tls client cert file.
-	RedisTLSClientCertFilename = "/certs/tls.crt"
+	RedisTLSClientCertFilename = "/redis-certs/tls.crt"
 	// RedisTLSClientKeyEnvVar is the redis tls client key.
 	RedisTLSClientKeyEnvVar = "REDIS_TLS_CLIENT_KEY"
 	// RedisTLSClientKeyFilename is the redis client key file.
-	RedisTLSClientKeyFilename = "/certs/tls.key"
+	RedisTLSClientKeyFilename = "/redis-certs/tls.key"
 	// RuntimeRootEnvVar is the runtime root.
 	RuntimeRootEnvVar = "RUNTIME_ROOT"
 	// RuntimeSubdirectoryEnvVar is the runtime subdirectory.
@@ -42,6 +42,20 @@ const (
 	RuntimeIgnoreDotfilesEnvVar = "RUNTIME_IGNOREDOTFILES"
 	// RuntimeWatchRootEnvVar is the runtime watch root.
 	RuntimeWatchRootEnvVar = "RUNTIME_WATCH_ROOT"
+	// GRPCServerUseTLSEnvVar is tls enable option for grpc server.
+	GRPCServerUseTLSEnvVar = "GRPC_SERVER_USE_TLS"
+	// GRPCServerTLSCertEnvVar is the grpc server tls cert.
+	GRPCServerTLSCertEnvVar = "GRPC_SERVER_TLS_CERT"
+	// GRPCServerTLSClientCertFilename is the GRPC tls cert file.
+	GRPCServerTLSCertFilename = "/certs/tls.crt"
+	// GRPCServerTLSKeyEnvVarEnvVar is the grpc server tls key.
+	GRPCServerTLSKeyEnvVarEnvVar = "GRPC_SERVER_TLS_KEY"
+	// GRPCServerTLSKeyFilename is the grpc server key file.
+	GRPCServerTLSKeyFilename = "/certs/tls.key"
+	// GRPCServerTLSCACertEnvVar is the grpc server tls ca cert.
+	GRPCServerTLSCACertEnvVar = "GRPC_SERVER_TLS_CA_CERT"
+	// GRPCServerTLSKeyFilename is the grpc server tls ca cert file.
+	GRPCServerTLSCACertFilename = "/certs/ca.crt"
 	// LogLevelEnvVar is the log level.
 	LogLevelEnvVar = "LOG_LEVEL"
 	// UseStatsdEnvVar is the use statsd.
@@ -113,10 +127,16 @@ func expectedContainerVolumeMounts(rateLimit *egcfgv1a1.RateLimit, rateLimitDepl
 	var volumeMounts []corev1.VolumeMount
 
 	// mount the cert
+	volumeMounts = append(volumeMounts, corev1.VolumeMount{
+		Name:      "certs",
+		MountPath: "/certs",
+		ReadOnly:  true,
+	})
+
 	if rateLimit.Backend.Redis.TLS != nil {
 		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      "certs",
-			MountPath: "/certs",
+			Name:      "redis-certs",
+			MountPath: "/redis-certs",
 			ReadOnly:  true,
 		})
 	}
@@ -130,7 +150,7 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme
 
 	if rateLimit.Backend.Redis.TLS != nil && rateLimit.Backend.Redis.TLS.CertificateRef != nil {
 		volumes = append(volumes, corev1.Volume{
-			Name: "certs",
+			Name: "redis-certs",
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: string(rateLimit.Backend.Redis.TLS.CertificateRef.Name),
@@ -139,6 +159,15 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme
 		})
 	}
 
+	volumes = append(volumes, corev1.Volume{
+		Name: "certs",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: "envoy-rate-limit",
+			},
+		},
+	})
+
 	return resource.ExpectedDeploymentVolumes(rateLimitDeployment.Pod, volumes)
 }
 
@@ -189,6 +218,22 @@ func expectedRateLimitContainerEnv(rateLimit *egcfgv1a1.RateLimit, rateLimitDepl
 			Name:  ConfigGrpcXdsNodeID,
 			Value: InfraName,
 		},
+		{
+			Name:  GRPCServerUseTLSEnvVar,
+			Value: "true",
+		},
+		{
+			Name:  GRPCServerTLSCertEnvVar,
+			Value: GRPCServerTLSCertFilename,
+		},
+		{
+			Name:  GRPCServerTLSKeyEnvVarEnvVar,
+			Value: GRPCServerTLSKeyFilename,
+		},
+		{
+			Name:  GRPCServerTLSCACertEnvVar,
+			Value: GRPCServerTLSCACertFilename,
+		},
 	}
 
 	if rateLimit.Backend.Redis.TLS != nil {

internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go

@@ -16,6 +16,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/utils/pointer"
+
 	gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 	"sigs.k8s.io/yaml"
 

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml

@@ -50,6 +50,14 @@ spec:
           value: envoy-gateway:18001
         - name: CONFIG_GRPC_XDS_NODE_ID
           value: envoy-ratelimit
+        - name: GRPC_SERVER_USE_TLS
+          value: "true"
+        - name: GRPC_SERVER_TLS_CERT
+          value: "/certs/tls.crt"
+        - name: GRPC_SERVER_TLS_KEY
+          value: "/certs/tls.key"
+        - name: GRPC_SERVER_TLS_CA_CERT
+          value: "/certs/ca.crt"
         image: custom-image
         imagePullPolicy: IfNotPresent
         name: envoy-ratelimit
@@ -68,6 +76,10 @@ spec:
             memory: 1Gi
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /certs
+          name: certs
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       schedulerName: default-scheduler
@@ -84,3 +96,7 @@ spec:
                 values:
                 - router-node
       terminationGracePeriodSeconds: 300
+      volumes:
+      - name: certs
+        secret:
+          secretName: envoy-rate-limit

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml

@@ -50,6 +50,14 @@ spec:
           value: envoy-gateway:18001
         - name: CONFIG_GRPC_XDS_NODE_ID
           value: envoy-ratelimit
+        - name: GRPC_SERVER_USE_TLS
+          value: "true"
+        - name: GRPC_SERVER_TLS_CERT
+          value: "/certs/tls.crt"
+        - name: GRPC_SERVER_TLS_KEY
+          value: "/certs/tls.key"
+        - name: GRPC_SERVER_TLS_CA_CERT
+          value: "/certs/ca.crt"
         image: custom-image
         imagePullPolicy: IfNotPresent
         name: envoy-ratelimit
@@ -68,10 +76,18 @@ spec:
             memory: 1Gi
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /certs
+          name: certs
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccountName: envoy-ratelimit
       securityContext:
         runAsUser: 1000
       terminationGracePeriodSeconds: 300
+      volumes:
+      - name: certs
+        secret:
+          secretName: envoy-rate-limit

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml

@@ -50,6 +50,14 @@ spec:
           value: envoy-gateway:18001
         - name: CONFIG_GRPC_XDS_NODE_ID
           value: envoy-ratelimit
+        - name: GRPC_SERVER_USE_TLS
+          value: "true"
+        - name: GRPC_SERVER_TLS_CERT
+          value: "/certs/tls.crt"
+        - name: GRPC_SERVER_TLS_KEY
+          value: "/certs/tls.key"
+        - name: GRPC_SERVER_TLS_CA_CERT
+          value: "/certs/ca.crt"
         image: custom-image
         imagePullPolicy: IfNotPresent
         name: envoy-ratelimit
@@ -68,10 +76,18 @@ spec:
             memory: 1Gi
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /certs
+          name: certs
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccountName: envoy-ratelimit
       securityContext:
         runAsUser: 1000
       terminationGracePeriodSeconds: 300
+      volumes:
+      - name: certs
+        secret:
+          secretName: envoy-rate-limit

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml

@@ -48,6 +48,14 @@ spec:
           value: envoy-gateway:18001
         - name: CONFIG_GRPC_XDS_NODE_ID
           value: envoy-ratelimit
+        - name: GRPC_SERVER_USE_TLS
+          value: "true"
+        - name: GRPC_SERVER_TLS_CERT
+          value: "/certs/tls.crt"
+        - name: GRPC_SERVER_TLS_KEY
+          value: "/certs/tls.key"
+        - name: GRPC_SERVER_TLS_CA_CERT
+          value: "/certs/ca.crt"
         image: envoyproxy/ratelimit:master
         imagePullPolicy: IfNotPresent
         name: envoy-ratelimit
@@ -61,8 +69,16 @@ spec:
             memory: 512Mi
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /certs
+          name: certs
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccountName: envoy-ratelimit
       terminationGracePeriodSeconds: 300
+      volumes:
+      - name: certs
+        secret:
+          secretName: envoy-rate-limit

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml

@@ -50,6 +50,14 @@ spec:
           value: envoy-gateway:18001
         - name: CONFIG_GRPC_XDS_NODE_ID
           value: envoy-ratelimit
+        - name: GRPC_SERVER_USE_TLS
+          value: "true"
+        - name: GRPC_SERVER_TLS_CERT
+          value: "/certs/tls.crt"
+        - name: GRPC_SERVER_TLS_KEY
+          value: "/certs/tls.key"
+        - name: GRPC_SERVER_TLS_CA_CERT
+          value: "/certs/ca.crt"
         - name: env_a
           value: env_a_value
         - name: env_b
@@ -72,10 +80,18 @@ spec:
             memory: 1Gi
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /certs
+          name: certs
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccountName: envoy-ratelimit
       securityContext:
         runAsUser: 1000
       terminationGracePeriodSeconds: 300
+      volumes:
+      - name: certs
+        secret:
+          secretName: envoy-rate-limit

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml

@@ -50,6 +50,14 @@ spec:
           value: envoy-gateway:18001
         - name: CONFIG_GRPC_XDS_NODE_ID
           value: envoy-ratelimit
+        - name: GRPC_SERVER_USE_TLS
+          value: "true"
+        - name: GRPC_SERVER_TLS_CERT
+          value: "/certs/tls.crt"
+        - name: GRPC_SERVER_TLS_KEY
+          value: "/certs/tls.key"
+        - name: GRPC_SERVER_TLS_CA_CERT
+          value: "/certs/ca.crt"
         image: custom-image
         imagePullPolicy: IfNotPresent
         name: envoy-ratelimit
@@ -68,10 +76,18 @@ spec:
             memory: 1Gi
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /certs
+          name: certs
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccountName: envoy-ratelimit
       securityContext:
         runAsUser: 1000
       terminationGracePeriodSeconds: 300
+      volumes:
+      - name: certs
+        secret:
+          secretName: envoy-rate-limit