semi working

.gitignore

@@ -0,0 +1,3 @@
+values.yaml
+.kcp
+*.kubeconfig

readme.md → README.md

@@ -2,6 +2,11 @@
 
 Repository for KCP helm charts.
 
+## Pre-requisites
+
+- Cert-manager installed and running
+- Ingress installed (e.g. nginx-ingress or OpenShift router)
+
 ## Usage
 
 [Helm](https://helm.sh) must be installed to use the charts.  Please refer to
@@ -22,3 +27,8 @@ To install the kcp chart:
 To uninstall the chart:
 
     helm delete my-kcp
+
+To install using local chart:
+
+    helm upgrade -i my-kcp ./charts/kcp/ --values values.yaml
+

charts/kcp/templates/kcp-front-proxy.yaml

@@ -238,6 +238,8 @@ spec:
         image: {{ .Values.kcpFrontProxy.image }}:{{ .Values.kcpFrontProxy.tag }}
         ports:
         - containerPort: 8443
+        command:
+        - /kcp-front-proxy
         args:
         - --secure-port=8443
         - --root-kubeconfig=/etc/kcp-front-proxy/kubeconfig/root-shard.kubeconfig
@@ -249,22 +251,6 @@ spec:
         {{- if .Values.kcpFrontProxy.profiling.enabled }}
         - --profiler-address=localhost:{{- .Values.kcpFrontProxy.profiling.port -}}
         {{- end }}
-        livenessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: livez
-            port: 8443
-            scheme: HTTPS
-          initialDelaySeconds: 45
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 10
-        readinessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: readyz
-            port: 8443
-            scheme: HTTPS
         resources:
           limits:
             cpu: 200m

charts/kcp/templates/kcp.yaml

@@ -31,6 +31,21 @@ spec:
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
+metadata:
+  name: kcp-service-accounts-ca
+spec:
+  isCA: true
+  commonName: kcp-service-accounts-ca
+  secretName: kcp-service-accounts-ca
+  privateKey:
+    {{- toYaml .Values.certificates.privateKeys | nindent 4 }}
+  issuerRef:
+    name: kcp-pki-ca
+    kind: Issuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
 metadata:
   name: kcp-server-client-ca
 spec:
@@ -205,11 +220,14 @@ spec:
         - --client-ca-file=/etc/kcp/tls/server-client/ca.crt
         - --tls-private-key-file=/etc/kcp/tls/server/tls.key
         - --tls-cert-file=/etc/kcp/tls/server/tls.crt
+        - --service-account-key-file=/etc/kcp/tls/service-accounts/ca.crt
+        - --service-account-private-key-file=/etc/kcp/tls/service-accounts/tls.key
         - --requestheader-client-ca-file=/etc/kcp/tls/requestheader-client/ca.crt
         - --requestheader-username-headers=X-Remote-User
         - --requestheader-group-headers=X-Remote-Group
-        - --root-directory=/etc/kcp/config
+        - --root-directory=/etc/kcp/config/kcp-0
         - --run-virtual-workspaces=false
+        - --shard-name=shard-1
         - --shard-base-url=https://kcp:6443
         - --shard-external-url=https://$(EXTERNAL_HOSTNAME):443
         - --external-hostname=$(EXTERNAL_HOSTNAME):443
@@ -223,6 +241,9 @@ spec:
         - "--oidc-username-prefix={{ .Values.oidc.usernamePrefix }}"
         - "--oidc-groups-prefix={{ .Values.oidc.groupsPrefix }}"
         {{- end }}
+        {{- if .Values.kcp.tokenAuth.enabled }}
+        - --token-auth-file=/etc/kcp/token-auth/{{ .Values.kcp.tokenAuth.fileName }}
+        {{- end }}
         - --v={{ .Values.kcp.v }}
         {{- if .Values.audit.enabled }}
         - --audit-log-maxage={{ .Values.audit.log.maxAge }}
@@ -281,10 +302,16 @@ spec:
           mountPath: /etc/kcp/tls/server
         - name: kcp-server-client-ca
           mountPath: /etc/kcp/tls/server-client
-        - name: kcp-requestheader-client-ca 
+        - name: kcp-service-accounts-ca
+          mountPath: /etc/kcp/tls/service-accounts
+        - name: kcp-requestheader-client-ca
           mountPath: /etc/kcp/tls/requestheader-client
         - name: kubeconfig
           mountPath: /etc/kcp/config
+        {{- if .Values.kcp.tokenAuth.enabled }}
+        - name: kcp-token-auth-file
+          mountPath: /etc/kcp/token-auth
+        {{- end}}
         {{- if .Values.audit.enabled }}
         - name: audit-log
           mountPath: {{ .Values.audit.log.dir }}
@@ -301,8 +328,8 @@ spec:
         - sh
         - -c
         - >
-          cat /etc/kcp/config/admin.kubeconfig | sed -e 's;://\([^/]*\);://localhost:6443;' -e 's;current-context: root;current-context: system:admin;' > /etc/kcp/config/localhost.kubeconfig &&
-          exec /usr/bin/virtual-workspaces
+          cat /etc/kcp/config/kcp-0/admin.kubeconfig | sed -e 's;://\([^/]*\);://localhost:6443;' -e 's;current-context: root;current-context: system:admin;' > /etc/kcp/config/localhost.kubeconfig &&
+          exec /virtual-workspaces
           workspaces
           --kubeconfig=/etc/kcp/config/localhost.kubeconfig
           --authentication-kubeconfig=/etc/kcp/config/localhost.kubeconfig
@@ -314,7 +341,7 @@ spec:
           --requestheader-username-headers=X-Remote-User
           --requestheader-group-headers=X-Remote-Group
           --secure-port=6444
-          --v={{ .Values.kcp.v }}
+          --v={{ .Values.virtualWorkspaces.v }}
           {{- if .Values.virtualWorkspaces.profiling.enabled }}
           --profiler-address=localhost:{{- .Values.virtualWorkspaces.profiling.port -}}
           {{- end }}
@@ -356,7 +383,7 @@ spec:
           mountPath: /etc/kcp/tls/server
         - name: kcp-server-client-ca
           mountPath: /etc/kcp/tls/server-client
-        - name: kcp-requestheader-client-ca 
+        - name: kcp-requestheader-client-ca
           mountPath: /etc/kcp/tls/requestheader-client
         - name: kubeconfig
           mountPath: /etc/kcp/config
@@ -376,6 +403,9 @@ spec:
           items:
           - key: ca.crt
             path: ca.crt
+      - name: kcp-service-accounts-ca
+        secret:
+          secretName: kcp-service-accounts-ca
       - name: kcp-server-client-ca
         secret:
           secretName: kcp-server-client-ca
@@ -393,6 +423,11 @@ spec:
         persistentVolumeClaim:
           claimName: kcp-audit-logs
       {{- end }}
+      {{- if .Values.kcp.tokenAuth.enabled }}
+      - name: kcp-token-auth-file
+        secret:
+          secretName: kcp-token-auth-file
+      {{- end }}
       - name: root-ca-file
         configMap:
           name: kcp-root-ca
@@ -404,3 +439,14 @@ metadata:
 data:
   {{ .Values.audit.policy.fileName }}: |
     {{- .Values.audit.policy.config | nindent 4 }}
+
+{{- if .Values.kcp.tokenAuth.enabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kcp-token-auth-file
+stringData:
+  {{ .Values.kcp.tokenAuth.fileName }}: |
+    {{- .Values.kcp.tokenAuth.config | nindent 4 }}
+{{- end}}

charts/kcp/values.yaml

@@ -4,16 +4,20 @@ etcd:
   image: quay.io/coreos/etcd
   tag: v3.5.4
   memoryLimit: 4Gi
+  memoryRequest: 1Gi
+  cpuLimit: 2
+  cpuRequest: 100m
   volumeSize: 8Gi
   profiling:
     enabled: false
 kcp:
   image: registry.ci.openshift.org/kcp/kcp
   tag: latest
-  v: "4"
+  v: "2"
   memoryLimit: 20Gi
-  cpuRequest: 500m
   memoryRequest: 4Gi
+  cpuLimit: 8
+  cpuRequest: 500m
   etcd:
     serverAddress: https://etcd:2379
     clientCertificate:
@@ -25,10 +29,19 @@ kcp:
   profiling:
     enabled: false
     port: 6060
+  tokenAuth:
+    enabled: false
+    fileName: auth-token.csv
+    config: |
+        user-1-token,user-1,1111-1111-1111-1111,"team-1"
+        #user-2-token,user-2,2222-2222-2222-2222,"team-2"
+        user-3-token,user-3,3333-3333-3333-3333,"team-3"
+        user-4-token,user-4,4444-4444-4444-4444,"team-4"
+        admin-token,admin,5555-5555-5555-5555,"system:masters"
 kcpFrontProxy:
   image: registry.ci.openshift.org/kcp/kcp-front-proxy
   tag: latest
-  v: "4"
+  v: "2"
   openshiftRoute:
     enabled: false
   ingress:
@@ -39,21 +52,23 @@ kcpFrontProxy:
     enabled: false
     className: ""
   certificate:
-    issuerSpec: {}
+    issuerSpec: {selfSigned: {}}
   profiling:
     enabled: false
     port: 6060
 virtualWorkspaces:
   image: registry.ci.openshift.org/kcp/virtual-workspaces
   tag: latest
+  v: "2"
   memoryLimit: 10Gi
-  cpuRequest: 100m
   memoryRequest: 2Gi
+  cpuLimit: 8
+  cpuRequest: 500m
   profiling:
     enabled: false
     port: 6061
 audit:
-  enabled: true
+  enabled: false
   volumeSize: 1Gi
   volumeClassName: ""
   policy:

hack/kind-config.yaml

@@ -0,0 +1,29 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+featureGates:
+  EphemeralContainers: true
+kubeadmConfigPatches:
+- |
+  apiVersion: kubeadm.k8s.io/v1beta2
+  kind: ClusterConfiguration
+  metadata:
+    name: config
+  apiServer:
+    extraArgs:
+      "enable-admission-plugins": NamespaceLifecycle,LimitRanger,ServiceAccount,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,PersistentVolumeClaimResize,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.24.2
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 443
+        hostPort: 443
+        protocol: TCP
+      - containerPort: 80
+        hostPort: 80
+        protocol: TCP

hack/setup-kind.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+
+if [ ! -f "/usr/local/bin/kind" ]; then
+ echo "Installing KIND"
+ curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.14.0/kind-linux-amd64
+ chmod +x ./kind
+ sudo mv ./kind /usr/local/bin/kind
+else
+    echo "KIND already installed"
+fi
+
+CLUSTER_NAME=faros
+
+if ! kind get clusters | grep -w -q "${CLUSTER_NAME}"; then
+kind create cluster --name kcp \
+     --kubeconfig .kcp/admin.kubeconfig \
+     --config ./hack/kind-config.yaml
+else
+    echo "Cluster already exists"
+fi
+
+export KUBECONFIG=.kcp/admin.kubeconfig
+
+echo "Installing ingress"
+
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+
+echo "Installing cert-manager"
+
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.crds.yaml
+helm install \
+  cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --create-namespace \
+  --version v1.9.1
+