add EKS Demo and trial support

README.md

@@ -1,6 +1,6 @@
 # Docker Registry Proxy
 
-This project provides a proxy to a Docker Registry with support for arbitrary authentication. We desire managed private Docker Registries, such as [Google Container Registry (GCR)](https://cloud.google.com/container-registry), [Amazon Elastic Container Registry (ECR)](https://aws.amazon.com/ecr/), [Private Docker Hub](https://docs.docker.com/docker-hub/repos/#private-repositories), [Quay.io](https://quay.io/), etc., as they are much cheaper to set-up, manage and reliably scale. However, authentication and access control options for these are limited. 
+This project provides a proxy to a Docker Registry with support for arbitrary authentication. We desire managed private Docker Registries, such as [Google Container Registry (GCR)](https://cloud.google.com/container-registry), [Amazon Elastic Container Registry (ECR)](https://aws.amazon.com/ecr/), [Private Docker Hub](https://docs.docker.com/docker-hub/repos/#private-repositories), [Quay.io](https://quay.io/), etc., as they are much cheaper to set-up, manage and reliably scale. However, authentication and access control options for these are limited.
 
 With this in mind, some use-cases of this Docker Registry Proxy are:
 
@@ -18,9 +18,5 @@ This is deployed in a container via a `DaemonSet`, thus requires access to the `
 ### Managed Kubernetes
 In most managed kubernetes offerings, it is difficult to modify the `kubelet` flags. The GKE documentation [recommends making host image modifications via a DaemonSet](https://cloud.google.com/kubernetes-engine/docs/concepts/node-images#modifications). This section lists the `DaemonSet` workarounds used for each Cloud Service Provider (CSP).
 
-#### Google Kubernetes Engine (GKE)
-The DaemonSet updates `/etc/default/kubelet` on the host (using `hostPath` mounts) and then kills the `kubelet` process on the host (via `hostPID: true`) which is then restarted by the host SystemD with the new kubelet flags.
-
-#### Amazon Elastic Kubernetes Engine (EKS)
-
-TODO
+#### Google Kubernetes Engine (GKE) and Amazon Elastic Kubernetes Engine (EKS)
+The DaemonSet updates `/etc/systemd/system/kubelet.service` on the host (using `hostPath` mounts) and then restarts `kubelet` via `systemd` (using `hostPath` mounts to `/run/systemd`).

deployments/demo/README.md

@@ -18,6 +18,6 @@ See `up.sh`.
 
 This is how to configure your Kubernetes workers to authenticate and pull images from the Docker Registry Proxy.
 
-This component creates a GCP Project `demo-kubelet-image-service-*` to demonstrate a simple deployment of the Kubelet Image Service and updating of Kubelet flags on Google Kubernetes Engine (GKE). There is also a `ghost-k8s` directory to deploy a Ghost CMS that references the Docker Registry Proxy.
+This component creates GKE and EKS clusters to demonstrate a simple deployment of the Kubelet Image Service and updating of Kubelet flags on Google Kubernetes Engine (GKE). There is also a `ghost-k8s` directory to deploy a Ghost CMS that references the Docker Registry Proxy.
 
 See `up.sh`.

deployments/demo/docker-registry-proxy/k8s/.gitignore

@@ -1,3 +1,3 @@
 gcp-sa.json
 ingress.yaml
-gcp-provider.yaml
+gcr.yaml

deployments/demo/docker-registry-proxy/k8s/ingress.yaml

@@ -1,11 +1,11 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: docker-registry-proxy
   namespace: docker-registry-proxy
   annotations:
     # add an annotation indicating the issuer to use.
-    cert-manager.io/cluster-issuer: letsencrypt
+    cert-manager.io/cluster-issuer: letsencrypt-prod
 spec:
   rules:
     - host: docker.vjpatel.me

deployments/demo/docker-registry-proxy/k8s/kustomization.yaml

@@ -7,7 +7,7 @@ resources:
   - ingress.yaml
 
 patches:
-  - gcp-provider.yaml
+  - gcr.yaml
 
 secretGenerator:
   - name: gcp-sa

deployments/demo/docker-registry-proxy/terraform/gcr.tf

@@ -1,6 +1,6 @@
 
 resource "google_project_service" "gcr_apis" {
-  project = "${google_project.my_project.project_id}"
+  project = google_project.my_project.project_id
   service = "containerregistry.googleapis.com"
 
   provisioner "local-exec" {
@@ -9,7 +9,7 @@ resource "google_project_service" "gcr_apis" {
 }
 
 resource "google_service_account" "service_account" {
-  project = "${google_project.my_project.project_id}"
+  project = google_project.my_project.project_id
 
   account_id   = "docker-registry-proxy"
   display_name = "Docker Registry Proxy"

deployments/demo/docker-registry-proxy/terraform/gke.tf

@@ -1,5 +1,5 @@
 resource "google_project_service" "gke_apis" {
-  project = "${google_project.my_project.project_id}"
+  project = google_project.my_project.project_id
   service = "container.googleapis.com"
 
   depends_on = ["google_project_service.gcr_apis"]
@@ -10,7 +10,7 @@ resource "google_project_service" "gke_apis" {
 }
 
 resource "google_container_cluster" "primary" {
-  project = "${google_project.my_project.project_id}"
+  project = google_project.my_project.project_id
 
   name     = "docker-registry-proxy"
   location = "europe-west1"
@@ -32,7 +32,7 @@ resource "google_container_cluster" "primary" {
 }
 
 resource "google_container_node_pool" "primary_preemptible_nodes" {
-  project = "${google_project.my_project.project_id}"
+  project = google_project.my_project.project_id
 
   name       = "preemptible-node-pool"
   location   = "europe-west1"

deployments/demo/docker-registry-proxy/up.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 ### SETTINGS
-acme_email="meetthevj@gmail.com"
+acme_email="vj@vjpatel.me"
 gcp_billing_account_id="00C32F-55AB5C-78A691"
 domain="docker.vjpatel.me"
 
@@ -27,44 +27,59 @@ project=$(jq -r '.resources[].instances[].attributes.project' terraform/terrafor
 
 gcloud container clusters get-credentials docker-registry-proxy --region europe-west1 --project "${project}"
 
-kubectl label nodes kubernetes.io/os=linux --all
+kubectl label nodes kubernetes.io/os=linux --all || true
 
-kubectl create namespace ingress-nginx
+kubectl create namespace ingress-nginx || true
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.28.0/deploy/static/mandatory.yaml
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.28.0/deploy/static/provider/cloud-generic.yaml
 
-kubectl create namespace cert-manager
+kubectl create namespace cert-manager || true
+kubectl apply --validate=false -f https://raw.githubusercontent.com/jetstack/cert-manager/v0.13.0/deploy/manifests/00-crds.yaml
 kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.13.0/cert-manager.yaml
 
-kubectl --namespace cert-manager wait --for condition=available deployment cert-manager-webhook
+kubectl --namespace cert-manager wait --for condition=available deployment cert-manager-webhook --timeout=300s
 
 cat <<EOF | kubectl apply -f-
 apiVersion: cert-manager.io/v1alpha2
 kind: ClusterIssuer
 metadata:
-  name: letsencrypt
+  name: letsencrypt-prod
 spec:
   acme:
     email: ${acme_email}
-    # server: https://acme-staging-v02.api.letsencrypt.org/directory
     server: https://acme-v02.api.letsencrypt.org/directory
     privateKeySecretRef:
-      name: letsencrypt-issuer-account-key
+      name: letsencrypt-prod-issuer-account-key
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-stag
+spec:
+  acme:
+    email: ${acme_email}
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: letsencrypt-stag-issuer-account-key
     solvers:
     - http01:
         ingress:
           class: nginx
 EOF
 
 cat <<EOF > k8s/ingress.yaml
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: docker-registry-proxy
   namespace: docker-registry-proxy
   annotations:
     # add an annotation indicating the issuer to use.
-    cert-manager.io/cluster-issuer: letsencrypt
+    cert-manager.io/cluster-issuer: letsencrypt-prod
 spec:
   rules:
     - host: ${domain}
@@ -80,7 +95,7 @@ spec:
       secretName: cert-${domain//./-} # < cert-manager will store the created certificate in this secret.
 EOF
 
-cat <<EOF > k8s/gcp-provider.yaml
+cat <<EOF > k8s/gcr.yaml
 ---
 kind: Deployment
 apiVersion: apps/v1
@@ -95,7 +110,7 @@ spec:
             - name: DRP_UPSTREAM_REPOSITORY
               value: https://gcr.io/${project}
             - name: DRP_PLUGINS
-              value: "auth-provider_static:upstream_static auth-verifier_gcp-instanceidentitydocument:gcpverifier"
+              value: "auth-provider_static:upstream_static auth-verifier_gcp-instanceidentitydocument:gcpverifier auth-verifier_aws-instanceidentitydocument:awsverifier"
             - name: DRP_UPSTREAM_AUTHENTICATION
               value: upstream_static
             - name: DRP_UPSTREAM_STATIC_USERNAME
@@ -106,9 +121,13 @@ spec:
                   name: gcp-sa
                   key: gcp-sa.json
             # - name: DRP_AUTHENTICATION_VERIFIER
-            #   value: _gcpidd:gcpverifier
+            #   value: "_gcpidd:gcpverifier _awsidd:awsverifier"
             # - name: DRP_GCPVERIFIER_PROJECT_IDS
             #   value: <project id>
+            # - name: DRP_AUTHENTICATION_VERIFIER
+            #   value: _awsidd:awsverifier
+            # - name: DRP_AWSVERIFIER_ACCOUNT_IDS
+            #   value: <account id>
 
 EOF
 
@@ -120,3 +139,30 @@ $ kubectl --namespace ingress-nginx get svc
 Deploy the docker registry proxy by running:
 $ kustomize build k8s/ | kubectl apply -f-
 EOF
+
+cat <<EOF > kubectl apply -f-
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager-test
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: test-selfsigned
+  namespace: cert-manager-test
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: selfsigned-cert
+  namespace: cert-manager-test
+spec:
+  dnsNames:
+    - example.com
+  secretName: selfsigned-cert-tls
+  issuerRef:
+    name: test-selfsigned
+EOF

deployments/demo/kubelet-image-service/k8s/eks/eks-provider.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kubelet-image-service
+  labels:
+    k8s-app: kubelet-image-service
+spec:
+  selector:
+    matchLabels:
+      name: kubelet-image-service
+  template:
+    metadata:
+      labels:
+        name: kubelet-image-service
+spec:
+  template:
+    spec:
+      containers:
+        - name: kubelet-image-service
+          env:
+            - name: DOCKER_API_VERSION
+              value: "1.38"
+            - name: KIS_PLUGINS
+              value: "auth-provider_aws-instanceidentitydocument:awsidd"
+            - name: KIS_AWSIDD_USERNAME
+              value: "_gcpidd"
+            - name: KIS_AWSIDD_AUDIENCE
+              value: "docker.vjpatel.me"
+            - name: KIS_AUTHENTICATION_PROVIDER
+              value: docker.vjpatel.me=awsidd

deployments/demo/kubelet-image-service/k8s/kubelet-patch.daemonset.yaml → deployments/demo/kubelet-image-service/k8s/eks/kubelet-patch.daemonset.yaml

@@ -18,45 +18,45 @@ spec:
       # tolerations:
       # - key: node-role.kubernetes.io/master
       #   effect: NoSchedule
-      hostPID: true
       containers:
         - name: kubelet-flag-patch
-          image: ubuntu:latest
-          command: ["/bin/bash", "-c"]
+          image: centos/systemd
+          command: ["bash", "-c"]
           args:
             - echo "wait for kubelet image service sock";
               while [ ! -S /var/run/kubelet/kubelet-image-service.sock ]; do sleep 1; done;
-              echo "original kubelet config";
-              cat /etc/default/kubelet;
-              cat /new/kubelet.conf > /etc/default/kubelet;
-              echo "new kubelet config";
-              cat /etc/default/kubelet;
-              echo "killing kubelet process to be restarted by host";
-              ps -eF | grep bin/kubelet | head -n1 | tr -s " " | cut -f2 -d" " | xargs kill -9;
-              sleep 999999999999999999999999999999999999
-          securityContext:
-            privileged: true
+              cat /etc/systemd/system/kubelet.service > /etc/systemd/system/kubelet.service.bak;
+              cat /new/kubelet.service > /etc/systemd/system/kubelet.service;
+              echo "reloading systemd and restarting kubelet";
+              systemctl daemon-reload && systemctl restart kubelet;
+              echo "sleeping forever";
+              while true; do sleep 60; done;
           resources:
             limits:
               memory: 16Mi
             requests:
               cpu: 16m
               memory: 8Mi
           volumeMounts:
-            - name: kubelet-config
-              mountPath: /etc/default/kubelet
-            - name: new-kubelet-config
-              mountPath: /new
             - name: kubelet-image-sock
               mountPath: /var/run/kubelet/
+            - name: systemd
+              mountPath: /run/systemd
+            - name: new-kubelet-systemd-cfg
+              mountPath: /new
+            - name: kubelet-systemd-cfg
+              mountPath: /etc/systemd/system/kubelet.service
       terminationGracePeriodSeconds: 30
       volumes:
-        - name: new-kubelet-config
-          configMap:
-            name: kubelet-config
-        - name: kubelet-config
-          hostPath:
-            path: /etc/default/kubelet
         - name: kubelet-image-sock
           hostPath:
             path: /var/run/kubelet/
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+        - name: new-kubelet-systemd-cfg
+          configMap:
+            name: kubelet-systemd-cfg
+        - name: kubelet-systemd-cfg
+          hostPath:
+            path: /etc/systemd/system/kubelet.service

deployments/demo/kubelet-image-service/k8s/eks/kubelet.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Kubernetes Kubelet
+Documentation=https://github.com/kubernetes/kubernetes
+After=docker.service
+Requires=docker.service
+
+[Service]
+ExecStartPre=/sbin/iptables -P FORWARD ACCEPT
+ExecStart=/usr/bin/kubelet --cloud-provider aws \
+    --config /etc/kubernetes/kubelet/kubelet-config.json \
+    --kubeconfig /var/lib/kubelet/kubeconfig \
+    --container-runtime docker \
+    --network-plugin cni \
+    --image-service-endpoint=unix:///var/run/kubelet/kubelet-image-service.sock \
+   $KUBELET_ARGS $KUBELET_EXTRA_ARGS
+
+Restart=on-failure
+RestartForceExitStatus=SIGPIPE
+RestartSec=5
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target