[WIP] Tide-up helm chart

.gitignore

@@ -0,0 +1,3 @@
+values.yaml
+.kcp
+*.kubeconfig

readme.md → README.md

@@ -2,6 +2,11 @@
 
 Repository for KCP helm charts.
 
+## Pre-requisites
+
+- Cert-manager installed and running
+- Ingress installed (e.g. nginx-ingress or OpenShift router)
+
 ## Usage
 
 [Helm](https://helm.sh) must be installed to use the charts.  Please refer to
@@ -22,3 +27,8 @@ To install the kcp chart:
 To uninstall the chart:
 
     helm delete my-kcp
+
+To install using local chart:
+
+    helm upgrade -i my-kcp ./charts/kcp/ --values values.yaml
+

charts/kcp/templates/kcp-front-proxy.yaml

@@ -27,19 +27,13 @@ metadata:
 spec:
   tls:
   - hosts:
-      - {{ .Values.externalHostname }}
+    - {{ .Values.externalHostname }}
     {{- with .Values.kcpFrontProxy.ingress.secret -}}secretName: {{ . }}{{- end }}
   rules:
   - host: {{ .Values.externalHostname }}
     http:
       paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: kcp-front-proxy
-            port:
-              number: 8443
+      {{- toYaml .Values.kcpFrontProxy.ingress.paths | nindent 6 }}
 {{- end }}
 {{- if .Values.kcpFrontProxy.gateway.enabled }}
 ---
@@ -206,15 +200,16 @@ data:
   path-mapping.yaml: |
     - path: /services/
       backend: https://kcp:6444
-      backend_server_ca: /etc/virtual-workspaces/tls/ca.crt
-      proxy_client_cert: /etc/kcp-front-proxy/requestheader-client/tls/virtual-workspaces/tls.crt
-      proxy_client_key: /etc/kcp-front-proxy/requestheader-client/tls/virtual-workspaces/tls.key
-    - path: /
-      backend: https://kcp:6443
+      backend_server_ca: /etc/kcp/tls/ca.crt
+      proxy_client_cert: /etc/kcp-front-proxy/requestheader-client/tls/kcp/tls.crt
+      proxy_client_key: /etc/kcp-front-proxy/requestheader-client/tls/kcp/tls.key
+    - path: /clusters/
+      backend: https://kcp:6444
       backend_server_ca: /etc/kcp/tls/ca.crt
       proxy_client_cert: /etc/kcp-front-proxy/requestheader-client/tls/kcp/tls.crt
       proxy_client_key: /etc/kcp-front-proxy/requestheader-client/tls/kcp/tls.key
 ---
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -238,9 +233,12 @@ spec:
         image: {{ .Values.kcpFrontProxy.image }}:{{ .Values.kcpFrontProxy.tag }}
         ports:
         - containerPort: 8443
+        command:
+        - /kcp-front-proxy
         args:
         - --secure-port=8443
-        - --root-kubeconfig=/etc/kcp-front-proxy/kubeconfig/root-shard.kubeconfig
+        - --root-kubeconfig=/etc/kcp-front-proxy/kubeconfig-root/root.kubeconfig
+        - --shards-kubeconfig=/etc/kcp-front-proxy/kubeconfig-shards/shards.kubeconfig
         - --tls-private-key-file=/etc/kcp-front-proxy/tls/tls.key
         - --tls-cert-file=/etc/kcp-front-proxy/tls/tls.crt
         - --client-ca-file=/etc/kcp-front-proxy/client/tls/ca.crt
@@ -279,8 +277,10 @@ spec:
           mountPath: /etc/kcp-front-proxy/config
         - name: client-cert-for-kubeconfig
           mountPath: /etc/kcp-front-proxy/client-cert-for-kubeconfig
-        - name: proxy-kcp-kubeconfig
-          mountPath: /etc/kcp-front-proxy/kubeconfig
+        - name: root-kcp-kubeconfig
+          mountPath: /etc/kcp-front-proxy/kubeconfig-root/
+        - name: shards-kcp-kubeconfig
+          mountPath: /etc/kcp-front-proxy/kubeconfig-shards/
         - name: kcp-client-ca
           mountPath: /etc/kcp-front-proxy/client/tls
         - name: kcp-ca
@@ -337,12 +337,18 @@ spec:
               path: tls.crt
             - key: tls.key
               path: tls.key
-      - name: proxy-kcp-kubeconfig
+      - name: root-kcp-kubeconfig
+        secret:
+          secretName: root-kcp-kubeconfig
+          items:
+            - key: kubeconfig
+              path: root.kubeconfig
+      - name: shards-kcp-kubeconfig
         secret:
-          secretName: proxy-kcp-kubeconfig
+          secretName: shards-kcp-kubeconfig
           items:
             - key: kubeconfig
-              path: root-shard.kubeconfig
+              path: shards.kubeconfig
       - name: kcp-front-proxy-config
         configMap:
           name: kcp-front-proxy-config

charts/kcp/templates/kcp.yaml

@@ -191,10 +191,15 @@ spec:
       labels:
         app: kcp
     spec:
+      {{- if .Values.kcp.hostAliases.enabled }}
+      hostAliases:
+        {{- toYaml .Values.kcp.hostAliases.values | nindent 6 }}
+      {{- end }}
       containers:
       - name: kcp
         image: {{ .Values.kcp.image }}:{{ .Values.kcp.tag }}
         ports:
+        - containerPort: 6444
         - containerPort: 6443
         args:
         - start
@@ -208,11 +213,10 @@ spec:
         - --requestheader-client-ca-file=/etc/kcp/tls/requestheader-client/ca.crt
         - --requestheader-username-headers=X-Remote-User
         - --requestheader-group-headers=X-Remote-Group
-        - --root-directory=/etc/kcp/config
-        - --run-virtual-workspaces=false
-        - --shard-base-url=https://kcp:6443
-        - --shard-external-url=https://$(EXTERNAL_HOSTNAME):443
+        - --secure-port=6444
+        - --shard-external-url=https://$(EXTERNAL_HOSTNAME)
         - --external-hostname=$(EXTERNAL_HOSTNAME):443
+        - --virtual-workspaces-workspaces.authorization-cache.resync-period=1s
         - --shard-virtual-workspace-url=https://$(EXTERNAL_HOSTNAME):443
         - --root-ca-file=/etc/kcp/tls/ca/root-ca.pem
         {{- if .Values.oidc }}
@@ -222,6 +226,12 @@ spec:
         - --oidc-username-claim={{ .Values.oidc.usernameClaim }}
         - "--oidc-username-prefix={{ .Values.oidc.usernamePrefix }}"
         - "--oidc-groups-prefix={{ .Values.oidc.groupsPrefix }}"
+        {{- if .Values.oidc.caSecretName }}
+        - --oidc-ca-file=/etc/kcp/tls/oidc/ca.crt
+        {{- end }}
+        {{- end }}
+        {{- if .Values.kcp.tokenAuth.enabled }}
+        - --token-auth-file=/etc/kcp/token-auth/{{ .Values.kcp.tokenAuth.fileName }}
         {{- end }}
         - --v={{ .Values.kcp.v }}
         {{- if .Values.audit.enabled }}
@@ -241,11 +251,15 @@ spec:
         env:
         - name: EXTERNAL_HOSTNAME
           value: {{ required "A valid external hostname is required" .Values.externalHostname }}
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
         livenessProbe:
           failureThreshold: 3
           httpGet:
             path: livez
-            port: 6443
+            port: 6444
             scheme: HTTPS
           initialDelaySeconds: 45
           periodSeconds: 10
@@ -254,7 +268,7 @@ spec:
         startupProbe:
           httpGet:
             path: readyz
-            port: 6443
+            port: 6444
             scheme: HTTPS
           initialDelaySeconds: 10
           timeoutSeconds: 1
@@ -265,7 +279,7 @@ spec:
           failureThreshold: 3
           httpGet:
             path: readyz
-            port: 6443
+            port: 6444
             scheme: HTTPS
         resources:
           limits:
@@ -281,10 +295,14 @@ spec:
           mountPath: /etc/kcp/tls/server
         - name: kcp-server-client-ca
           mountPath: /etc/kcp/tls/server-client
-        - name: kcp-requestheader-client-ca 
+        - name: kcp-requestheader-client-ca
           mountPath: /etc/kcp/tls/requestheader-client
         - name: kubeconfig
           mountPath: /etc/kcp/config
+        {{- if .Values.kcp.tokenAuth.enabled }}
+        - name: kcp-token-auth-file
+          mountPath: /etc/kcp/token-auth
+        {{- end}}
         {{- if .Values.audit.enabled }}
         - name: audit-log
           mountPath: {{ .Values.audit.log.dir }}
@@ -293,73 +311,10 @@ spec:
         {{- end }}
         - name: root-ca-file
           mountPath: /etc/kcp/tls/ca
-      - name: virtual-workspaces
-        image: {{ .Values.virtualWorkspaces.image }}:{{ .Values.virtualWorkspaces.tag }}
-        ports:
-        - containerPort: 6444
-        command:
-        - sh
-        - -c
-        - >
-          cat /etc/kcp/config/admin.kubeconfig | sed -e 's;://\([^/]*\);://localhost:6443;' -e 's;current-context: root;current-context: system:admin;' > /etc/kcp/config/localhost.kubeconfig &&
-          exec /usr/bin/virtual-workspaces
-          workspaces
-          --kubeconfig=/etc/kcp/config/localhost.kubeconfig
-          --authentication-kubeconfig=/etc/kcp/config/localhost.kubeconfig
-          --authentication-skip-lookup
-          --client-ca-file=/etc/kcp/tls/server-client/ca.crt
-          --tls-private-key-file=/etc/kcp/tls/server/tls.key
-          --tls-cert-file=/etc/kcp/tls/server/tls.crt
-          --requestheader-client-ca-file=/etc/kcp/tls/requestheader-client/ca.crt
-          --requestheader-username-headers=X-Remote-User
-          --requestheader-group-headers=X-Remote-Group
-          --secure-port=6444
-          --v={{ .Values.kcp.v }}
-          {{- if .Values.virtualWorkspaces.profiling.enabled }}
-          --profiler-address=localhost:{{- .Values.virtualWorkspaces.profiling.port -}}
-          {{- end }}
-        livenessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: livez
-            port: 6444
-            scheme: HTTPS
-          initialDelaySeconds: 45
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 10
-        startupProbe:
-          httpGet:
-            path: readyz
-            port: 6444
-            scheme: HTTPS
-          initialDelaySeconds: 10
-          timeoutSeconds: 1
-          periodSeconds: 10
-          successThreshold: 1
-          failureThreshold: 18
-        readinessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: readyz
-            port: 6444
-            scheme: HTTPS
-        resources:
-          limits:
-            cpu: '{{ .Values.virtualWorkspaces.cpuLimit }}'
-            memory: '{{ .Values.virtualWorkspaces.memoryLimit }}'
-          requests:
-            cpu: '{{ .Values.virtualWorkspaces.cpuRequest }}'
-            memory: '{{ .Values.virtualWorkspaces.memoryRequest }}'
-        volumeMounts:
-        - name: virtual-workspaces-certs
-          mountPath: /etc/kcp/tls/server
-        - name: kcp-server-client-ca
-          mountPath: /etc/kcp/tls/server-client
-        - name: kcp-requestheader-client-ca 
-          mountPath: /etc/kcp/tls/requestheader-client
-        - name: kubeconfig
-          mountPath: /etc/kcp/config
+        {{- if .Values.oidc.caSecretName }}
+        - name: oidc-ca
+          mountPath: /etc/kcp/tls/oidc
+        {{- end }}
       volumes:
       - name: etcd-certs
         secret:
@@ -393,9 +348,19 @@ spec:
         persistentVolumeClaim:
           claimName: kcp-audit-logs
       {{- end }}
+      {{- if .Values.kcp.tokenAuth.enabled }}
+      - name: kcp-token-auth-file
+        secret:
+          secretName: kcp-token-auth-file
+      {{- end }}
       - name: root-ca-file
         configMap:
           name: kcp-root-ca
+      {{- if .Values.oidc.caSecretName }}
+      - name: oidc-ca
+        secret:
+          secretName: {{.Values.oidc.caSecretName }}
+      {{- end }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -404,3 +369,14 @@ metadata:
 data:
   {{ .Values.audit.policy.fileName }}: |
     {{- .Values.audit.policy.config | nindent 4 }}
+
+{{- if .Values.kcp.tokenAuth.enabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kcp-token-auth-file
+stringData:
+  {{ .Values.kcp.tokenAuth.fileName }}: |
+    {{- .Values.kcp.tokenAuth.config | nindent 4 }}
+{{- end}}

charts/kcp/templates/front-proxy-kubeconfig.yaml → charts/kcp/templates/root-kubeconfig.yaml

@@ -5,7 +5,7 @@ stringData:
     clusters:
     - cluster:
         certificate-authority: /etc/kcp/tls/ca.crt
-        server: https://kcp:6443
+        server: https://kcp:6444
       name: system:admin
     contexts:
     - context:
@@ -22,5 +22,5 @@ stringData:
         client-key: /etc/kcp-front-proxy/client-cert-for-kubeconfig/tls.key
 kind: Secret
 metadata:
-  name: proxy-kcp-kubeconfig
+  name: root-kcp-kubeconfig
 type: Opaque

charts/kcp/templates/shards-kubeconfig.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+stringData:
+  kubeconfig: |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority: /etc/kcp/tls/ca.crt
+        server: https://kcp:6444
+      name: system:admin
+    contexts:
+    - context:
+        cluster: system:admin
+        user: admin
+      name: system:admin
+    current-context: system:admin
+    kind: Config
+    preferences: {}
+    users:
+    - name: admin
+      user:
+        client-certificate: /etc/kcp-front-proxy/client-cert-for-kubeconfig/tls.crt
+        client-key: /etc/kcp-front-proxy/client-cert-for-kubeconfig/tls.key
+kind: Secret
+metadata:
+  name: shards-kcp-kubeconfig
+type: Opaque