Ensure everything is secret. Move into charts

README.md

@@ -24,9 +24,9 @@ helm upgrade --namespace=wandb --create-namespace --install wandb oci://us-centr
 Then provision your instance with:
 
 ```shell
-git clone https://github.com/wandb/charts.git
-cd charts/wandb
-helm upgrade --namespace=wandb --create-namespace --install wandb . --set license=$LICENSE --set bucket=$BUCKET --set bucketRegion=$BUCKET_REGION
+git clone https://github.com/wandb/helm-charts.git
+cd helm-charts
+helm upgrade --namespace=wandb --create-namespace --install wandb ./charts/wandb --set license=$LICENSE --set bucket=$BUCKET --set bucketRegion=$BUCKET_REGION
 ```
 
 ## Releasing

wandb/templates/NOTES.txt → charts/wandb/templates/NOTES.txt

@@ -11,3 +11,11 @@ To access W&B, run:
   kubectl port-forward svc/wandb 8080
 
 {{- end }}
+
+{{- if not .Values.bucket }}
+
+WARNING: You haven't specified an external object storage solution.  Data will 
+  be persisted on the node running our container but all data will be lost if
+  this node goes away.
+
+{{- end }}

wandb/templates/deployment.yaml → charts/wandb/templates/deployment.yaml

@@ -64,7 +64,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: wandb-secrets
-                  key: license
+                  key: LICENSE
             - name: WANDB_HELM_CHART
               value: {{ include "wandb.fullname" . }}:{{ .Chart.Version }}
             {{- if .Values.smtpServer }}
@@ -76,7 +76,10 @@ spec:
             {{- end }}
             {{- if .Values.bucket }}
             - name: BUCKET
-              value: {{ .Values.bucket }}
+              valueFrom:
+                secretKeyRef:
+                  name: wandb-secrets
+                  key: BUCKET
             - name: AWS_REGION
               value: {{ .Values.bucketRegion | default .Release.Namespace }}
             - name: AWS_S3_KMS_ID
@@ -112,6 +115,11 @@ spec:
               path: /ready
               port: http
             failureThreshold: 60
+          {{- if not .Values.bucket }}
+          volumeMounts:
+            - name: wandb-data
+              mountPath: /vol
+          {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -125,4 +133,10 @@ spec:
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}
+      {{- if not .Values.bucket }}
+      volumes:
+        - name: wandb-data
+          hostPath:
+            path: /wandb
+      {{- end }}

wandb/templates/mysql.yaml → charts/wandb/templates/mysql.yaml

@@ -18,6 +18,7 @@ spec:
       {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
+        # TODO: ideally only create the checksum when the DB password is rotated
         checksum/config: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
       labels:
         {{- include "wandb.selectorLabels" . | nindent 8 }}
@@ -92,7 +93,12 @@ spec:
           {{- if .Values.mysql.persistence.subPath }}
           subPath: {{ .Values.mysql.persistence.subPath }}
           {{- end }}
+        - name: mysql-initdb
+          mountPath: /docker-entrypoint-initdb.d
       volumes:
+      - name: mysql-initdb
+        configMap:
+          name: mysql-initdb-config
       - name: data
       {{- if .Values.mysql.persistence.enabled }}
         persistentVolumeClaim:
@@ -101,6 +107,15 @@ spec:
         emptyDir: {}
       {{- end }}
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql-initdb-config
+data:
+  # We need RELOAD for making backups
+  initdb.sql: |
+    GRANT RELOAD ON *.* TO `wandb_local`@`%`;
+---
 {{- if and .Values.mysql.persistence.enabled (not .Values.mysql.persistence.existingClaim) }}
 kind: PersistentVolumeClaim
 apiVersion: v1

wandb/templates/secrets.yaml → charts/wandb/templates/secrets.yaml

@@ -10,14 +10,16 @@ type: Opaque
 # TODO: figure out a better way to support rotating
 data:
 {{- if $secret }}
-  license: {{ $secret.data.license }}
   MYSQL_ROOT_PASSWORD: {{ $secret.data.MYSQL_ROOT_PASSWORD }}
   MYSQL_PASSWORD: {{ $secret.data.MYSQL_PASSWORD }}
 {{- else }}
-  license: {{ required "license must be specified to deploy with helm" .Values.license | b64enc }}
   MYSQL_ROOT_PASSWORD: {{ randAlphaNum 10 | b64enc }}
   MYSQL_PASSWORD: {{ default (randAlphaNum 10) .Values.mysql.password | b64enc }}
 {{- end }}
+  LICENSE: {{ required "license must be specified to deploy with helm" .Values.license | b64enc }}
 {{- if .Values.smtpServer }}
   SMTP_SERVER: {{ .Values.smtpServer | b64enc }}
-{{- end }}
+{{- end }}
+{{- if .Values.bucket }}
+  BUCKET: {{ .Values.bucket | b64enc }}
+{{- end}}

wandb/values.yaml → charts/wandb/values.yaml

@@ -40,13 +40,9 @@ serviceAccount:
 podAnnotations: {}
 
 podSecurityContext:
-  {}
-  # fsGroup: 2000
+  fsGroup: 0
 
 securityContext:
-  capabilities:
-    drop:
-      - ALL
   runAsNonRoot: true
   runAsUser: 100000
   runAsGroup: 0

dev-kind-cluster.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -o errexit
+
+# create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+# create a cluster with the local registry enabled in containerd
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_name}:5000"]
+EOF
+
+# connect the registry to the cluster network if not already connected
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+echo <<EOF Tag your images with localhost:${reg_port} and push:
+
+  docker build . -t wandb/local:0.9.49
+  docker tag wandb/local:0.9.49 localhost:${reg_port}/wandb/local:0.9.49
+  docker push localhost:${reg_port}/wandb/local:0.9.49
+EOF