Allow the Controller and Webhook Containers to run with the securityC…

…ontext: readOnlyRootfilesystem: true (#2282) * create a tmp dir for the controller to write Spark artifacts to and set the controller to readOnlyRootFilesystem Signed-off-by: Nick Gretzon <npgretz@gmail.com> * mount a dir for the webhook container to generate its certificates in and set readOnlyRootFilesystem: true for the webhook pod Signed-off-by: Nick Gretzon <npgretz@gmail.com> * update the securityContext in the controller deployment test Signed-off-by: Nick Gretzon <npgretz@gmail.com> * update securityContext of the webhook container in the deployment_test Signed-off-by: Nick Gretzon <npgretz@gmail.com> * update README Signed-off-by: Nick Gretzon <npgretz@gmail.com> * remove -- so comments are not rendered in the README.md Signed-off-by: Nick Gretzon <npgretz@gmail.com> * recreate README.md after removal of comments for volumes and volumeMounts Signed-off-by: Nick Gretzon <npgretz@gmail.com> * make indentation for volumes and volumeMounts consistent with rest of values.yaml Signed-off-by: Nick Gretzon <npgretz@gmail.com> * Revert "make indentation for volumes and volumeMounts consistent with rest of values.yaml" This reverts commit dba97fc. Signed-off-by: Nick Gretzon <npgretz@gmail.com> * fix indentation in webhook and controller deployment templates for volumes and volumeMounts Signed-off-by: Nick Gretzon <npgretz@gmail.com> * Update charts/spark-operator-chart/values.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * Update charts/spark-operator-chart/values.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * Update charts/spark-operator-chart/values.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * Update charts/spark-operator-chart/values.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * Update charts/spark-operator-chart/templates/controller/deployment.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * Update charts/spark-operator-chart/templates/controller/deployment.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * Update charts/spark-operator-chart/templates/webhook/deployment.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * Update charts/spark-operator-chart/templates/webhook/deployment.yaml Co-authored-by: Yi Chen <github@chenyicn.net> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> * add additional securityContext to the controller deployment_test.yaml Signed-off-by: Nick Gretzon <npgretz@gmail.com> --------- Signed-off-by: Nick Gretzon <npgretz@gmail.com> Signed-off-by: Nicholas Gretzon <50811947+npgretz@users.noreply.github.com> Co-authored-by: Yi Chen <github@chenyicn.net>
kubeflow · Nov 7, 2024 · 72107fd · 72107fd
1 parent 763682d
commit 72107fd
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 12 deletions.
diff --git a/charts/spark-operator-chart/README.md b/charts/spark-operator-chart/README.md
@@ -102,7 +102,7 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | controller.rbac.annotations | object | `{}` | Extra annotations for the controller RBAC resources. |
 | controller.labels | object | `{}` | Extra labels for controller pods. |
 | controller.annotations | object | `{}` | Extra annotations for controller pods. |
-| controller.volumes | list | `[]` | Volumes for controller pods. |
+| controller.volumes | list | `[{"emptyDir":{"sizeLimit":"1Gi"},"name":"tmp"}]` | Volumes for controller pods. |
 | controller.nodeSelector | object | `{}` | Node selector for controller pods. |
 | controller.affinity | object | `{}` | Affinity for controller pods. |
 | controller.tolerations | list | `[]` | List of node taints to tolerate for controller pods. |
@@ -111,9 +111,9 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | controller.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). The labelSelector field in topology spread constraint will be set to the selector labels for controller pods if not specified. |
 | controller.env | list | `[]` | Environment variables for controller containers. |
 | controller.envFrom | list | `[]` | Environment variable sources for controller containers. |
-| controller.volumeMounts | list | `[]` | Volume mounts for controller containers. |
+| controller.volumeMounts | list | `[{"mountPath":"/tmp","name":"tmp","readOnly":false}]` | Volume mounts for controller containers. |
 | controller.resources | object | `{}` | Pod resource requests and limits for controller containers. Note, that each job submission will spawn a JVM within the controller pods using "/usr/local/openjdk-11/bin/java -Xmx128m". Kubernetes may kill these Java processes at will to enforce resource limits. When that happens, you will see the following error: 'failed to run spark-submit for SparkApplication [...]: signal: killed' - when this happens, you may want to increase memory limits. |
-| controller.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"runAsNonRoot":true}` | Security context for controller containers. |
+| controller.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true}` | Security context for controller containers. |
 | controller.sidecars | list | `[]` | Sidecar containers for controller pods. |
 | controller.podDisruptionBudget.enable | bool | `false` | Specifies whether to create pod disruption budget for controller. Ref: [Specifying a Disruption Budget for your Application](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) |
 | controller.podDisruptionBudget.minAvailable | int | `1` | The number of pods that must be available. Require `controller.replicas` to be greater than 1 |
@@ -141,7 +141,7 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | webhook.labels | object | `{}` | Extra labels for webhook pods. |
 | webhook.annotations | object | `{}` | Extra annotations for webhook pods. |
 | webhook.sidecars | list | `[]` | Sidecar containers for webhook pods. |
-| webhook.volumes | list | `[]` | Volumes for webhook pods. |
+| webhook.volumes | list | `[{"emptyDir":{"sizeLimit":"500Mi"},"name":"serving-certs"}]` | Volumes for webhook pods. |
 | webhook.nodeSelector | object | `{}` | Node selector for webhook pods. |
 | webhook.affinity | object | `{}` | Affinity for webhook pods. |
 | webhook.tolerations | list | `[]` | List of node taints to tolerate for webhook pods. |
@@ -150,9 +150,9 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | webhook.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). The labelSelector field in topology spread constraint will be set to the selector labels for webhook pods if not specified. |
 | webhook.env | list | `[]` | Environment variables for webhook containers. |
 | webhook.envFrom | list | `[]` | Environment variable sources for webhook containers. |
-| webhook.volumeMounts | list | `[]` | Volume mounts for webhook containers. |
+| webhook.volumeMounts | list | `[{"mountPath":"/etc/k8s-webhook-server/serving-certs","name":"serving-certs","readOnly":false,"subPath":"serving-certs"}]` | Volume mounts for webhook containers. |
 | webhook.resources | object | `{}` | Pod resource requests and limits for webhook pods. |
-| webhook.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"runAsNonRoot":true}` | Security context for webhook containers. |
+| webhook.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true}` | Security context for webhook containers. |
 | webhook.podDisruptionBudget.enable | bool | `false` | Specifies whether to create pod disruption budget for webhook. Ref: [Specifying a Disruption Budget for your Application](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) |
 | webhook.podDisruptionBudget.minAvailable | int | `1` | The number of pods that must be available. Require `webhook.replicas` to be greater than 1 |
 | spark.jobNamespaces | list | `["default"]` | List of namespaces where to run spark jobs. If empty string is included, all namespaces will be allowed. Make sure the namespaces have already existed. |

diff --git a/charts/spark-operator-chart/templates/webhook/deployment.yaml b/charts/spark-operator-chart/templates/webhook/deployment.yaml
@@ -94,7 +94,7 @@ spec:
         {{- end }}
         {{- with .Values.webhook.volumeMounts }}
         volumeMounts:
-          {{- toYaml . | nindent 10 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
         {{- with .Values.webhook.resources }}
         resources:
@@ -123,7 +123,7 @@ spec:
       {{- end }}
       {{- with .Values.webhook.volumes }}
       volumes:
-        {{- toYaml . | nindent 8 }}
+      {{- toYaml . | nindent 6 }}
       {{- end }}
       {{- with .Values.webhook.nodeSelector }}
       nodeSelector:

diff --git a/charts/spark-operator-chart/tests/controller/deployment_test.yaml b/charts/spark-operator-chart/tests/controller/deployment_test.yaml
@@ -355,16 +355,30 @@ tests:
     set:
       controller:
         securityContext:
+          readOnlyRootFilesystem: true
           runAsUser: 1000
           runAsGroup: 2000
           fsGroup: 3000
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          runAsNonRoot: true
+          privileged: false
     asserts:
       - equal:
           path: spec.template.spec.containers[0].securityContext
           value:
+            readOnlyRootFilesystem: true
             runAsUser: 1000
             runAsGroup: 2000
             fsGroup: 3000
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            runAsNonRoot: true
+            privileged: false
 
   - it: Should add sidecars if `controller.sidecars` is set
     set:

diff --git a/charts/spark-operator-chart/tests/webhook/deployment_test.yaml b/charts/spark-operator-chart/tests/webhook/deployment_test.yaml
@@ -299,10 +299,14 @@ tests:
     set:
       webhook:
         securityContext:
+          readOnlyRootFilesystem: true
           runAsUser: 1000
           runAsGroup: 2000
           fsGroup: 3000
     asserts:
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
+          value: true
       - equal:
           path: spec.template.spec.containers[0].securityContext.runAsUser
           value: 1000

diff --git a/charts/spark-operator-chart/values.yaml b/charts/spark-operator-chart/values.yaml
@@ -107,7 +107,11 @@ controller:
     # key2: value2
 
   # -- Volumes for controller pods.
-  volumes: []
+  volumes:
+  # Create a tmp directory to write Spark artifacts to for deployed Spark apps.
+  - name: tmp
+    emptyDir:
+      sizeLimit: 1Gi
 
   # -- Node selector for controller pods.
   nodeSelector: {}
@@ -143,7 +147,11 @@ controller:
   envFrom: []
 
   # -- Volume mounts for controller containers.
-  volumeMounts: []
+  volumeMounts:
+  # Mount a tmp directory to write Spark artifacts to for deployed Spark apps.
+  - name: tmp
+    mountPath: "/tmp"
+    readOnly: false
 
   # -- Pod resource requests and limits for controller containers.
   # Note, that each job submission will spawn a JVM within the controller pods using "/usr/local/openjdk-11/bin/java -Xmx128m".
@@ -159,6 +167,7 @@ controller:
 
   # -- Security context for controller containers.
   securityContext:
+    readOnlyRootFilesystem: true
     privileged: false
     allowPrivilegeEscalation: false
     runAsNonRoot: true
@@ -256,7 +265,11 @@ webhook:
   sidecars: []
 
   # -- Volumes for webhook pods.
-  volumes: []
+  volumes:
+  # Create a dir for the webhook to generate its certificates in.
+  - name: serving-certs
+    emptyDir:
+      sizeLimit: 500Mi
 
   # -- Node selector for webhook pods.
   nodeSelector: {}
@@ -292,7 +305,13 @@ webhook:
   envFrom: []
 
   # -- Volume mounts for webhook containers.
-  volumeMounts: []
+  volumeMounts:
+  # Mount a dir for the webhook to generate its certificates in.
+  - name: serving-certs
+    mountPath: /etc/k8s-webhook-server/serving-certs
+    subPath: serving-certs
+    readOnly: false
+
 
   # -- Pod resource requests and limits for webhook pods.
   resources: {}
@@ -305,6 +324,7 @@ webhook:
 
   # -- Security context for webhook containers.
   securityContext:
+    readOnlyRootFilesystem: true
     privileged: false
     allowPrivilegeEscalation: false
     runAsNonRoot: true