feat: support proxy configuration (env vars & CA certificate) (#137)

* fix: add yaml tags to ValidatorConfig types Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * feat: support proxy config Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * feat: support proxy config Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * chore: add Make target Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * chore: update chart README.md Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * fix: include root CAs and ca-certificates.conf in certs-init image Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * fix: configure root CAs properly Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * refactor: copy env & proxy config into each plugin's values Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * feat: support proxy CA cert secret creation Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * fix: use non-root user for cert-init image Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> * chore: update network plugin version Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com> --------- Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com>
validator-labs · Nov 29, 2023 · 63c3bc8 · 63c3bc8
1 parent 31746d6
commit 63c3bc8
Show file tree

Hide file tree

Showing 7 changed files with 132 additions and 34 deletions.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,7 @@
 
 # Image URL to use all building/pushing image targets
 IMG ?= quay.io/spectrocloud-labs/validator:latest
+CERTS_INIT_IMG ?= quay.io/spectrocloud-labs/validator-certs-init:latest
 
 GOARCH ?= $(shell go env GOARCH)
 
@@ -81,6 +82,10 @@ run: manifests generate fmt vet ## Run a controller from your host.
 docker-build: test ## Build docker image with the manager.
 	$(CONTAINER_TOOL) build -t ${IMG} . --platform linux/$(GOARCH)
 
+.PHONY: docker-build-certs-init
+docker-build-certs-init: ## Build validator-certs-init docker image.
+	$(CONTAINER_TOOL) build -f hack/validator-certs-init.Dockerfile -t ${CERTS_INIT_IMG} . --platform linux/$(GOARCH)
+
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG}

diff --git a/chart/validator/README.md b/chart/validator/README.md
@@ -2,7 +2,7 @@
 Validator
 ===========
 
-validator monitors ValidationResults created by one or more validator plugins and uploads them to a configurable sink
+Monitor results created by validator plugins and upload them to a configurable sink
 
 
 ## Configuration
@@ -15,26 +15,37 @@ The following table lists the configurable parameters of the Validator chart and
 | `controllerManager.kubeRbacProxy.containerSecurityContext.allowPrivilegeEscalation` |  | `false` |
 | `controllerManager.kubeRbacProxy.containerSecurityContext.capabilities.drop` |  | `["ALL"]` |
 | `controllerManager.kubeRbacProxy.image.repository` |  | `"gcr.io/kubebuilder/kube-rbac-proxy"` |
-| `controllerManager.kubeRbacProxy.image.tag` |  | `"v0.14.1"` |
+| `controllerManager.kubeRbacProxy.image.tag` |  | `"v0.15.0"` |
 | `controllerManager.kubeRbacProxy.resources.limits.cpu` |  | `"500m"` |
 | `controllerManager.kubeRbacProxy.resources.limits.memory` |  | `"128Mi"` |
 | `controllerManager.kubeRbacProxy.resources.requests.cpu` |  | `"5m"` |
 | `controllerManager.kubeRbacProxy.resources.requests.memory` |  | `"64Mi"` |
-| `controllerManager.manager.args` |  | `["--health-probe-bind-address=:8081", "--metrics-bind-address=127.0.0.1:8080", "--leader-elect"]` |
+| `controllerManager.manager.args` |  | `["--health-probe-bind-address=:8081", "--leader-elect"]` |
 | `controllerManager.manager.containerSecurityContext.allowPrivilegeEscalation` |  | `false` |
 | `controllerManager.manager.containerSecurityContext.capabilities.drop` |  | `["ALL"]` |
 | `controllerManager.manager.image.repository` |  | `"quay.io/spectrocloud-labs/validator"` |
 | `controllerManager.manager.image.tag` | x-release-please-version | `"v0.0.26"` |
 | `controllerManager.manager.resources.limits.cpu` |  | `"500m"` |
-| `controllerManager.manager.resources.limits.memory` |  | `"128Mi"` |
+| `controllerManager.manager.resources.limits.memory` |  | `"512Mi"` |
 | `controllerManager.manager.resources.requests.cpu` |  | `"10m"` |
 | `controllerManager.manager.resources.requests.memory` |  | `"64Mi"` |
+| `controllerManager.manager.sinkWebhookTimeout` |  | `"30s"` |
 | `controllerManager.replicas` |  | `1` |
 | `controllerManager.serviceAccount.annotations` |  | `{}` |
 | `kubernetesClusterDomain` |  | `"cluster.local"` |
 | `metricsService.ports` |  | `[{"name": "https", "port": 8443, "protocol": "TCP", "targetPort": "https"}]` |
 | `metricsService.type` |  | `"ClusterIP"` |
-| `plugins` |  | `[{"chart": {"name": "validator-plugin-aws", "repository": "https://spectrocloud-labs.github.io/validator-plugin-aws", "version": "v0.0.2"}, "values": "controllerManager:\n  kubeRbacProxy:\n    args:\n    - --secure-listen-address=0.0.0.0:8443\n    - --upstream=http://127.0.0.1:8080/\n    - --logtostderr=true\n    - --v=0\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: gcr.io/kubebuilder/kube-rbac-proxy\n      tag: v0.14.1\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 5m\n        memory: 64Mi\n  manager:\n    args:\n    - --health-probe-bind-address=:8081\n    - --metrics-bind-address=127.0.0.1:8080\n    - --leader-elect\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: quay.io/spectrocloud-labs/validator-plugin-aws\n      tag: v0.0.2\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 10m\n        memory: 64Mi\n  replicas: 1\n  serviceAccount:\n    annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n  ports:\n  - name: https\n    port: 8443\n    protocol: TCP\n    targetPort: https\n  type: ClusterIP\nauth:\n  secretName: aws-creds\n  accessKeyId: \"\"\n  secretAccessKey: \"\""}]` |
+| `env` |  | `[]` |
+| `proxy.enabled` |  | `false` |
+| `proxy.image` |  | `"quay.io/spectrocloud-labs/validator-certs-init:latest"` |
+| `proxy.secretName` |  | `"proxy-cert"` |
+| `proxy.createSecret` |  | `false` |
+| `proxy.caCert` | Raw CA certificate, required if createSecret is true | `""` |
+| `sink` |  | `{}` |
+| `plugins` |  | `[{"chart": {"name": "validator-plugin-aws", "repository": "https://spectrocloud-labs.github.io/validator-plugin-aws", "version": "v0.0.18"}, "values": "controllerManager:\n  kubeRbacProxy:\n    args:\n    - --secure-listen-address=0.0.0.0:8443\n    - --upstream=http://127.0.0.1:8080/\n    - --logtostderr=true\n    - --v=0\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: gcr.io/kubebuilder/kube-rbac-proxy\n      tag: v0.15.0\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 5m\n        memory: 64Mi\n  manager:\n    args:\n    - --health-probe-bind-address=:8081\n    - --leader-elect\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: quay.io/spectrocloud-labs/validator-plugin-aws\n      tag: v0.0.18\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 10m\n        memory: 64Mi\n  replicas: 1\n  serviceAccount:\n    annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n  ports:\n  - name: https\n    port: 8443\n    protocol: TCP\n    targetPort: https\n  type: ClusterIP\nauth:\n  # Option 1: Leave secret undefined for implicit auth (node instance IAM role, IMDSv2, etc.)\n  # Option 2: Create a secret via pluginSecrets (see below). Note: secretName and pluginSecrets.aws.secretName must match.\n  # Option 3: Specify the name of a preexisting secret in your target cluster and leave pluginSecrets.aws undefined.\n  #\n  secret: {}  # Delete these curly braces if you're specifying secretName!\n    # secretName: aws-creds\n\n  # Override the service account used by AWS validator (optional, could be used for IMDSv2 on EKS)\n  # WARNING: the chosen service account must include all RBAC privileges found in the AWS plugin template:\n  #          https://github.com/spectrocloud-labs/validator-plugin-aws/blob/main/chart/validator-plugin-aws/templates/manager-rbac.yaml\n  serviceAccountName: \"\""}, {"chart": {"name": "validator-plugin-azure", "repository": "https://spectrocloud-labs.github.io/validator-plugin-azure", "version": "v0.0.2"}, "values": "controllerManager:\n  kubeRbacProxy:\n    args:\n    - --secure-listen-address=0.0.0.0:8443\n    - --upstream=http://127.0.0.1:8080/\n    - --logtostderr=true\n    - --v=0\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: gcr.io/kubebuilder/kube-rbac-proxy\n      tag: v0.15.0\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 5m\n        memory: 64Mi\n  manager:\n    args:\n    - --health-probe-bind-address=:8081\n    - --leader-elect\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: quay.io/spectrocloud-labs/validator-plugin-azure\n      tag: v0.0.2\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 10m\n        memory: 64Mi\n    # Optionally specify a volumeMount to mount a volume containing a private key\n    # to leverage Azure Service principal with certificate authentication.\n    volumeMounts: []\n  replicas: 1\n  serviceAccount:\n    annotations: {}\n  # Optionally specify a volume containing a private key to leverage Azure Service\n  # principal with certificate authentication.\n  volumes: []\nkubernetesClusterDomain: cluster.local\nmetricsService:\n  ports:\n  - name: https\n    port: 8443\n    protocol: TCP\n    targetPort: https\n  type: ClusterIP\nauth:\n  # Option 1: Leave secret undefined for WorkloadIdentityCredential authentication.\n  # Option 2: Create a secret via pluginSecrets (see below). Note: secretName and pluginSecrets.azure.secretName must match.\n  # Option 3: Specify the name of a preexisting secret in your target cluster and leave pluginSecrets.azure undefined.\n  #\n  secret: {}  # Delete these curly braces if you're specifying secretName!\n    # secretName: azure-creds\n\n  # Override the service account used by Azure validator (optional, could be used for WorkloadIdentityCredentials on AKS)\n  # WARNING: the chosen service account must include all RBAC privileges found in the Azure plugin template:\n  #          https://github.com/spectrocloud-labs/validator-plugin-aws/blob/main/chart/validator-plugin-azure/templates/manager-rbac.yaml\n  serviceAccountName: \"\""}, {"chart": {"name": "validator-plugin-vsphere", "repository": "https://spectrocloud-labs.github.io/validator-plugin-vsphere", "version": "v0.0.13"}, "values": "controllerManager:\n  kubeRbacProxy:\n    args:\n    - --secure-listen-address=0.0.0.0:8443\n    - --upstream=http://127.0.0.1:8080/\n    - --logtostderr=true\n    - --v=0\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: gcr.io/kubebuilder/kube-rbac-proxy\n      tag: v0.14.1\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 5m\n        memory: 64Mi\n  manager:\n    args:\n    - --health-probe-bind-address=:8081\n    - --leader-elect\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: quay.io/spectrocloud-labs/validator-plugin-vsphere\n      tag: v0.0.13\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 10m\n        memory: 64Mi\n  replicas: 1\n  serviceAccount:\n    annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n  ports:\n  - name: https\n    port: 8443\n    protocol: TCP\n    targetPort: https\n  type: ClusterIP\nauth:\n  # Option 1: Create a secret via pluginSecrets (see below). Note: secretName and pluginSecrets.vSphere.secretName must match.\n  # Option 2: Specify the name of a preexisting secret in your target cluster and leave pluginSecrets.vSphere undefined.\n  secretName: vsphere-creds"}, {"chart": {"name": "validator-plugin-network", "repository": "https://spectrocloud-labs.github.io/validator-plugin-network", "version": "v0.0.9"}, "values": "controllerManager:\n  kubeRbacProxy:\n    args:\n    - --secure-listen-address=0.0.0.0:8443\n    - --upstream=http://127.0.0.1:8080/\n    - --logtostderr=true\n    - --v=0\n    containerSecurityContext:\n      allowPrivilegeEscalation: false\n      capabilities:\n        drop:\n        - ALL\n    image:\n      repository: gcr.io/kubebuilder/kube-rbac-proxy\n      tag: v0.15.0\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 5m\n        memory: 64Mi\n  manager:\n    args:\n    - --health-probe-bind-address=:8081\n    - --leader-elect\n    containerSecurityContext:\n      allowPrivilegeEscalation: true\n      capabilities:\n        add:\n        - NET_RAW\n        drop:\n        - ALL\n    image:\n      repository: quay.io/spectrocloud-labs/validator-plugin-network\n      tag: v0.0.9\n    resources:\n      limits:\n        cpu: 500m\n        memory: 128Mi\n      requests:\n        cpu: 10m\n        memory: 64Mi\n  replicas: 1\n  serviceAccount:\n    annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n  ports:\n  - name: https\n    port: 8443\n    protocol: TCP\n    targetPort: https\n  type: ClusterIP"}]` |
+| `pluginSecrets.aws` | Don't forget to delete these curly braces if you're specifying credentials here! | `{}` |
+| `pluginSecrets.azure` | Don't forget to delete these curly braces if you're specifying credentials here! | `{}` |
+| `pluginSecrets.vSphere` | Don't forget to delete these curly braces if you're specifying credentials here! | `{}` |
 
 
 

diff --git a/chart/validator/templates/deployment.yaml b/chart/validator/templates/deployment.yaml
@@ -22,22 +22,35 @@ spec:
       annotations:
         kubectl.kubernetes.io/default-container: manager
     spec:
+      {{- if .Values.proxy.enabled }}
+      initContainers:
+      - name: init-certs
+        image: {{ required ".Values.proxy.image is required!" .Values.proxy.image }}
+        command: ["/bin/bash", "-c"]
+        args: ["update-ca-certificates && cp -r /etc/ca-certificates.conf /usr/share/ca-certificates /etc/ssl/certs"]
+        volumeMounts:
+        - name: ca-pemstore
+          mountPath: /usr/local/share/ca-certificates
+          readOnly: false
+        - name: certs
+          mountPath: /etc/ssl/certs
+          readOnly: false
+        securityContext:
+          runAsNonRoot: false
+      {{- end }}
       containers:
       - args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }}
         env:
         - name: KUBERNETES_CLUSTER_DOMAIN
           value: {{ quote .Values.kubernetesClusterDomain }}
-        image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag
-          | default .Chart.AppVersion }}
+        image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag | default .Chart.AppVersion }}
         name: kube-rbac-proxy
         ports:
         - containerPort: 8443
           name: https
           protocol: TCP
-        resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent
-          10 }}
-        securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext
-          | nindent 10 }}
+        resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent 10 }}
+        securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext | nindent 10 }}
       - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }}
         command:
         - /manager
@@ -48,10 +61,12 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+          {{- if .Values.env }}
+          {{- toYaml .Values.env | nindent 8 }}
+          {{- end }}
         - name: SINK_WEBHOOK_TIMEOUT_SECONDS
           value: {{ quote .Values.controllerManager.manager.sinkWebhookTimeout }}
-        image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag
-          | default .Chart.AppVersion }}
+        image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag | default .Chart.AppVersion }}
         livenessProbe:
           httpGet:
             path: /healthz
@@ -65,13 +80,33 @@ spec:
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
-        resources: {{- toYaml .Values.controllerManager.manager.resources | nindent 10
-          }}
-        securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext
-          | nindent 10 }}
+        resources: {{- toYaml .Values.controllerManager.manager.resources | nindent 10 }}
+        securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext | nindent 10 }}
+        {{- if .Values.proxy.enabled }}
+        volumeMounts:
+        - name: certs
+          mountPath: /etc/ca-certificates.conf
+          subPath: ca-certificates.conf
+          readOnly: true
+        - name: certs
+          mountPath: /usr/share/ca-certificates
+          subPath: ca-certificates
+          readOnly: true
+        - name: certs
+          mountPath: /etc/ssl/certs
+          readOnly: true
+        {{- end }}
       securityContext:
         runAsNonRoot: true
         seccompProfile:
           type: RuntimeDefault
       serviceAccountName: {{ include "chart.fullname" . }}-controller-manager
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 10
+      {{- if .Values.proxy.enabled }}
+      volumes:
+      - name: ca-pemstore
+        secret:
+          secretName: {{ required ".Values.proxy.secretName is required!" .Values.proxy.secretName }}
+      - name: certs
+        emptyDir: {}
+      {{- end}}
diff --git a/chart/validator/templates/proxy-secret.yaml b/chart/validator/templates/proxy-secret.yaml
@@ -0,0 +1,9 @@
+{{- if and .Values.proxy.enabled .Values.proxy.createSecret }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ required ".Values.proxy.secretName is required!" .Values.proxy.secretName }}
+stringData:
+  ca.crt: |
+{{ required ".Values.proxy.caCert is required!" .Values.proxy.caCert | indent 4 }}
+{{- end }}
diff --git a/chart/validator/templates/validator-config.yaml b/chart/validator/templates/validator-config.yaml
@@ -1,10 +1,23 @@
+{{ $env := .Values.env }}
+{{ $proxy := .Values.proxy }}
 apiVersion: validation.spectrocloud.labs/v1alpha1
 kind: ValidatorConfig
 metadata:
   name: validator-config
 spec:
   plugins:
-{{ toYaml .Values.plugins | indent 2 }}
+{{- range .Values.plugins }}
+  -
+{{- toYaml . | nindent 4 }}
+      {{- if $env }}
+      env:
+      {{- toYaml $env | nindent 8 }}
+      {{- else }}
+      env: []
+      {{- end }}
+      proxy:
+      {{- toYaml $proxy | nindent 8 }}
+{{- end }}
 {{- if .Values.sink }}
   sink:
     type: {{ required ".Values.sink.type is required!" .Values.sink.type }}