Add cert-based API server authentication

Add the ability to authenticate incoming requests, verifying that all
requests originate from the Kubernetes API server and no where else.

Authenticating the API server requires manual steps to configure both
the API server and the webhook. Follow the Kubernetes webhook
documentation[1] to create an admission configuration and kubeconfig for
the API server, and update the kube-apiserver flags to use them. Only
cert-based authentication is supported, basic auth and token
authentication will not be recognized. Then, set auth.clientCA in the
webhook chart's values.yaml to the base64-encoded CA for the certs, and
set auth.allowedCNs to the CN for the client cert the apiserver will
present.

[1] https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#authenticate-apiservers

README.md

@@ -167,6 +167,9 @@ export CATTLE_WEBHOOK_URL="https://<NGROK_URL>.ngrok.io"
 ./bin/webhook
 ```
 After 15 seconds the webhook will update the `ValidatingWebhookConfiguration` and `MutatingWebhookConfiguration` in the Kubernetes cluster to point at the locally running instance.
+
+> :warning: Kubernetes API server authentication will not work with ngrok.
+
 ## License
 Copyright (c) 2019-2021 [Rancher Labs, Inc.](http://rancher.com)
 

charts/rancher-webhook/templates/deployment.yaml

@@ -1,3 +1,4 @@
+{{- $auth := .Values.auth | default dict }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -11,12 +12,19 @@ spec:
       labels:
         app: rancher-webhook
     spec:
-      {{- if .Values.capi.enabled }}
+      {{- if or .Values.capi.enabled $auth.clientCA }}
       volumes:
+      {{- end }}
+      {{- if .Values.capi.enabled }}
       - name: tls
         secret:
           secretName: rancher-webhook-tls
       {{- end }}
+      {{- if $auth.clientCA }}
+      - name: client-ca
+        secret:
+          secretName: client-ca
+      {{- end }}
       {{- if .Values.global.hostNetwork }}
       hostNetwork: true
       {{- end }}
@@ -44,6 +52,10 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        {{- if $auth.allowedCNs }}
+        - name: ALLOWED_CNS
+          value: '{{ join "," $auth.allowedCNs }}'
+        {{- end }}
         image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}'
         name: rancher-webhook
         imagePullPolicy: "{{ .Values.image.imagePullPolicy }}"
@@ -65,19 +77,26 @@ spec:
             port: "https"
             scheme: "HTTPS"
           periodSeconds: 5
-        {{- if .Values.capi.enabled }}
+        {{- if or .Values.capi.enabled $auth.clientCA }}
         volumeMounts:
+        {{- end }}
+        {{- if .Values.capi.enabled }}
         - name: tls
           mountPath: /tmp/k8s-webhook-server/serving-certs
+          readOnly: true
+        {{- end }}
+        {{- if $auth.clientCA }}
+        - name: client-ca
+          mountPath: /tmp/k8s-webhook-server/client-ca
+          readOnly: true
         {{- end }}
         {{- if .Values.capNetBindService }}
         securityContext:
           capabilities:
             add:
-            - NET_BIND_SERVICE 
+            - NET_BIND_SERVICE
         {{- end }}
       serviceAccountName: rancher-webhook
       {{- if .Values.priorityClassName }}
       priorityClassName: "{{.Values.priorityClassName}}"
       {{- end }}
-

charts/rancher-webhook/templates/secret.yaml

@@ -0,0 +1,11 @@
+{{- $auth := .Values.auth | default dict }}
+{{- if $auth.clientCA }}
+apiVersion: v1
+data:
+  ca.crt: {{ $auth.clientCA }}
+kind: Secret
+metadata:
+  name: client-ca
+  namespace: cattle-system
+type: Opaque
+{{- end }}

charts/rancher-webhook/tests/deployment_test.yaml

@@ -60,3 +60,35 @@ tests:
       - contains:
           path: spec.template.spec.containers[0].securityContext.capabilities.add
           content: NET_BIND_SERVICE
+
+  - it: should not set volumes or volumeMounts by default
+    asserts:
+      - isNull:
+          path: spec.template.spec.volumes
+      - isNull:
+          path: spec.template.spec.volumeMounts
+
+  - it: should set CA fields when CA options are set
+    set:
+      auth.clientCA: base64-encoded-cert
+      auth.allowedCNs:
+        - kube-apiserver
+        - joe
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: client-ca
+            secret:
+              secretName: client-ca
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: client-ca
+            mountPath: /tmp/k8s-webhook-server/client-ca
+            readOnly: true
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: ALLOWED_CNS
+            value: kube-apiserver,joe

charts/rancher-webhook/values.yaml

@@ -24,3 +24,11 @@ priorityClassName: ""
 
 # port assigns which port to use when running rancher-webhook
 port: 9443
+
+# Parameters for authenticating the kube-apiserver.
+auth:
+  # CA for authenticating kube-apiserver client certs. If empty, client connections will not be authenticated.
+  # Must be base64-encoded.
+  clientCA: ""
+  # Allowlist of CNs for kube-apiserver client certs. If empty, any cert signed by the CA provided in clientCA will be accepted.
+  allowedCNs: []

main.go

@@ -30,6 +30,9 @@ func run() error {
 	if os.Getenv("CATTLE_DEBUG") == "true" || os.Getenv("RANCHER_DEBUG") == "true" {
 		logrus.SetLevel(logrus.DebugLevel)
 	}
+	if os.Getenv("CATTLE_TRACE") == "true" {
+		logrus.SetLevel(logrus.TraceLevel)
+	}
 
 	logrus.Infof("Rancher-webhook version %s is starting", fmt.Sprintf("%s (%s)", Version, GitCommit))
 

pkg/server/server.go

@@ -4,9 +4,14 @@ package server
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 	"os"
+	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -38,8 +43,11 @@ const (
 	defaultWebhookHTTPSPort = 9443
 	webhookPortEnvKey       = "CATTLE_PORT"
 	webhookURLEnvKey        = "CATTLE_WEBHOOK_URL"
+	allowedCNsEnv           = "ALLOWED_CNS"
 )
 
+var caFile = filepath.Join(os.TempDir(), "k8s-webhook-server", "client-ca", "ca.crt")
+
 // tlsOpt option function applied to all webhook servers.
 var tlsOpt = func(config *tls.Config) {
 	config.MinVersion = tls.VersionTLS12
@@ -51,6 +59,7 @@ var tlsOpt = func(config *tls.Config) {
 		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
 		tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 	}
+	config.ClientAuth = tls.RequestClientCert
 }
 
 // ListenAndServe starts the webhook server.
@@ -118,6 +127,7 @@ func listenAndServe(ctx context.Context, clients *clients.Clients, validators []
 	router := mux.NewRouter()
 	errChecker := health.NewErrorChecker("Config Applied")
 	health.RegisterHealthCheckers(router, errChecker)
+	router.Use(certAuth())
 
 	logrus.Debug("Creating Webhook routes")
 	for _, webhook := range validators {
@@ -283,3 +293,80 @@ func (s *secretHandler) ensureWebhookConfiguration(validatingConfig *v1.Validati
 
 	return nil
 }
+
+// certAuth returns a middleware for cert-based authentication.
+// This is done as a middleware instead of using tls.RequireAndVerifyClientCert because an exception
+// needs to be made for the unauthenticated /healthz endpoint.
+func certAuth() func(next http.Handler) http.Handler {
+	opts := getVerifyOptions()
+	allowedCNs := getAllowedCNs()
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			logrus.Tracef("running cert check middleware for request %s", r.URL.Path)
+			if opts == nil {
+				next.ServeHTTP(w, r)
+				return
+			}
+			if r.URL.Path == "/healthz" { // apiserver does not present client cert for health checks
+				next.ServeHTTP(w, r)
+				return
+			}
+			if len(r.TLS.PeerCertificates) == 0 {
+				logrus.Warn("client did not present certificates")
+				http.Error(w, "could not verify client certificates", http.StatusUnauthorized)
+				return
+			}
+			for _, cert := range r.TLS.PeerCertificates[1:] {
+				opts.Intermediates.AddCert(cert)
+			}
+			_, err := r.TLS.PeerCertificates[0].Verify(*opts)
+			if err != nil {
+				logrus.Warnf("could not verify client certificates: %v", err)
+				http.Error(w, "could not verify client certificates", http.StatusUnauthorized)
+				return
+			}
+			if len(allowedCNs) == 0 {
+				next.ServeHTTP(w, r)
+				return
+			}
+			requestCN := r.TLS.PeerCertificates[0].Subject.CommonName
+			found := false
+			for _, allowed := range allowedCNs {
+				if allowed == requestCN {
+					found = true
+					break
+				}
+			}
+			if !found {
+				logrus.Warnf("could not find common name %s in allowed list", requestCN)
+				http.Error(w, "common name is not allowed", http.StatusUnauthorized)
+				return
+			}
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+func getVerifyOptions() *x509.VerifyOptions {
+	caCert, err := ioutil.ReadFile(caFile)
+	if err != nil {
+		logrus.Infof("could not read client CA file at %s, incoming requests will not be authenticated", caFile)
+		return nil
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+	opts := x509.VerifyOptions{
+		Roots:         caCertPool,
+		Intermediates: x509.NewCertPool(),
+		KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+	return &opts
+}
+
+func getAllowedCNs() []string {
+	allowedCNString := os.Getenv(allowedCNsEnv)
+	if len(allowedCNString) == 0 {
+		return nil
+	}
+	return strings.Split(allowedCNString, ",")
+}