Reproducible Teleport demo environments in Kubernetes

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "e"]
 	path = e
 	url = git@github.com:gravitational/teleport.e.git
+[submodule "examples/chart/teleport-demo/secrets"]
+	path = examples/chart/teleport-demo/secrets
+	url = git@github.com:gravitational/ops.git

examples/chart/teleport-demo/.gitignore

@@ -0,0 +1,3 @@
+license/
+secrets*.yaml
+secrets*.yaml.dec

examples/chart/teleport-demo/.helmignore

@@ -0,0 +1,5 @@
+*~
+*.pem
+scripts
+pki
+secrets

examples/chart/teleport-demo/Chart.yaml

@@ -0,0 +1,14 @@
+name: teleport-demo
+version: 0.0.5
+description: Teleport Enterprise
+keywords:
+  - Teleport Enterprise
+tillerVersion: ">=2.8.0"
+kubeVersion: ">=1.10.0-0"
+home: https://github.com/gravitational/teleport
+sources:
+  - https://github.com/gravitational/teleport
+maintainers:
+  - name: Gus Luxton
+    email: gus@gravitational.com
+    url: https://github.com/webvictim

examples/chart/teleport-demo/README.md

@@ -0,0 +1,106 @@
+# Teleport on Kubernetes
+
+[Gravitational Teleport](https://github.com/gravitational/teleport) is a modern SSH/Kubernetes API proxy server for
+remotely accessing clusters of Linux containers and servers via SSH, HTTPS, or Kubernetes API.
+
+This configuration is quite a Gravitational-specific deployment but should show a good amount of reusability for other
+savvy admins.
+
+## Introduction
+
+This chart deploys Teleport components to your cluster using various Kubernetes primitives.
+
+It supports a few key features:
+- A configurable number of nodes per cluster (n)
+- One 'main' cluster with <n> nodes in its own Kubernetes namespace
+- Any amount of different-named trusted clusters with <n> nodes, each in their own Kubernetes namespace
+    - These clusters are automatically linked to 'main' as trusted clusters
+- OIDC authentication via Auth0
+- DNS records pointing to a Kubernetes LoadBalancer for each cluster, set up on a configurable Cloudflare domain
+- LetsEncrypt certificates automatically provisioned, configured and renewed for each cluster via certbot-dns-cloudflare
+- Secrets encrypted using sops and a key from GKE
+
+See the comments in the default `values.yaml` and also the [Teleport documentation](https://gravitational.com/teleport/docs/quickstart) for more options.
+
+## Prerequisites
+
+- Kubernetes 1.10+
+- [sops](https://github.com/mozilla/sops)
+- [helm-secrets](https://github.com/futuresimple/helm-secrets)
+- [gcloud SDK](https://cloud.google.com/sdk/docs/downloads-interactive)
+    - ```curl https://sdk.cloud.google.com | bash``` for a simple install
+- Secrets stored in secrets.yaml and encrypted with sops
+    - Teleport Enterprise license
+    - Email address and API key for a Cloudflare account that controls the domain you wish to use
+    - Client ID and client secret for a configured Auth0 application
+
+## Installing the chart
+
+If you want to use a different version of Teleport, you should build and push the Docker images for the specified
+version to GCR:
+
+```
+$ cd examples/chart/teleport-demo/docker
+$ gcloud auth login
+$ gcloud auth configure-docker
+$ ./build-all.sh 3.1.8
+```
+
+Make sure that you have access to the key for sops encryption:
+```bash
+$ gcloud auth application-default login
+$ gcloud kms keys list --location global --keyring teleport-sops
+NAME                                                                                          PURPOSE          LABELS  PRIMARY_ID  PRIMARY_STATE
+projects/kubeadm-167321/locations/global/keyRings/teleport-sops/cryptoKeys/teleport-sops-key  ENCRYPT_DECRYPT          1           ENABLED
+```
+
+kubectl needs to know about your cluster - for GKE you can use something like this:
+
+```bash
+$ gcloud container clusters get-credentials <cluster-name> --zone <zone> --project <project>
+$ ./gke-init.sh
+```
+
+Make sure that you have updated the submodule containing the secrets:
+
+```bash
+$ git pull --recurse-submodules
+```
+
+To install the chart with the release name `teleport` and Teleport version 3.1.8, run:
+
+```bash
+$ helm secrets install --name teleport -f secrets/sops/teleport-demo/secrets.yaml ./ --set teleportVersion=3.1.8
+```
+
+Once the chart is installed successfully, Helm will output a section titled NOTES containing the URL to access the main
+cluster's web UI, along with some example `tsh` commands based on your installation.
+
+You can show these notes again in future with the `helm status <releaseName>` command - e.g. `helm status teleport`
+
+## Deleting the chart
+
+If you named the chart `teleport`:
+
+```bash
+$ helm delete --purge teleport
+```
+
+Namespaces will automatically be deleted once the cluster is shut down.
+
+## Recreating this without access to secrets
+
+If you're looking to use/modify this code and don't have access to the repo containing the sops-encrypted secrets,
+here's the sections you'll need to ensure you have in your `secrets.yaml` or equivalent file:
+
+```yaml
+secrets:
+  auth0:
+    client_id: <Auth0 client ID>
+    client_secret: <Auth0 client secret>
+  cloudflare:
+    api_key: <Cloudflare API key>
+    email: <Cloudflare email address>
+  license: |
+    <PEM-encoded Teleport enterprise license file>
+```

examples/chart/teleport-demo/docker/build-all.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -e
+VERSION=3.2.0
+if [[ "$1" != "" ]]; then
+    VERSION=$1
+    shift
+fi
+set -e
+for f in *; do
+    if [[ -d $f ]]; then
+        pushd $f
+        ./build.sh ${VERSION} "$@"
+        popd
+    fi
+done

examples/chart/teleport-demo/docker/cloudflare-agent/Dockerfile

@@ -0,0 +1,25 @@
+ARG TELEPORT_VERSION
+FROM quay.io/gravitational/debian-grande:latest
+
+ARG KUBECTL_VERSION="v1.12.5"
+ARG CURL_OPTS="-L --retry 100 --retry-delay 0 --connect-timeout 10 --max-time 300"
+
+# Update packages
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && \
+    apt-get -y install curl jq python2.7 build-essential python-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt
+
+# install kubectl
+RUN curl ${CURL_OPTS} https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl --output /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl
+
+# Install certbot to get/rotate certificates, add certbot-dns-cloudflare for registration
+RUN curl ${CURL_OPTS} -O https://bootstrap.pypa.io/get-pip.py && \
+    python2.7 get-pip.py && \
+    pip install certbot certbot-dns-cloudflare
+
+COPY rootfs/ /
+
+ENTRYPOINT ["/usr/bin/dumb-init", "/scripts/cloudflare-agent.sh"]

examples/chart/teleport-demo/docker/cloudflare-agent/build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -e
+VERSION=3.2.0
+if [[ "$1" != "" ]]; then
+    VERSION=$1
+    shift
+fi
+docker pull quay.io/gravitational/debian-grande:latest
+docker build --pull \
+    -t gcr.io/kubeadm-167321/cloudflare-agent:${VERSION} \
+    -t gcr.io/kubeadm-167321/cloudflare-agent:latest \
+    --cache-from quay.io/gravitational/debian-grande:latest,gcr.io/kubeadm-167321/cloudflare-agent:latest \
+    . $*
+docker push gcr.io/kubeadm-167321/cloudflare-agent:${VERSION}
+docker push gcr.io/kubeadm-167321/cloudflare-agent:latest

examples/chart/teleport-demo/docker/cloudflare-agent/rootfs/scripts/cloudflare-agent.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -e
+if [[ "${DEBUG}" == true ]]; then
+    set -x
+fi
+
+function cloudflareagent_log() {
+    echo "[cloudflare-agent] $*"
+}
+
+API_KEY=$(cat /etc/cloudflare/api_key)
+EMAIL=$(cat /etc/cloudflare/email)
+DOMAIN_TO_REGISTER="${CLUSTER_NAME}.${CLOUDFLARE_DOMAIN}"
+
+TLS_ENABLED=$(cat /etc/teleport-tls/enabled)
+LETSENCRYPT_ENABLED=$(cat /etc/teleport-tls/letsencrypt-enabled)
+
+if [[ "${DEBUG}" == true ]]; then
+    cloudflareagent_log "Cloudflare credentials:"
+    cloudflareagent_log "API key: ${API_KEY}"
+    cloudflareagent_log "Email: ${EMAIL}"
+    cloudflareagent_log "----"
+    cloudflareagent_log "Cluster name: ${CLUSTER_NAME}"
+    cloudflareagent_log "Domain: ${CLOUDFLARE_DOMAIN}"
+    cloudflareagent_log "Register: ${DOMAIN_TO_REGISTER}"
+    cloudflareagent_log "Cloudflare TTL: ${CLOUDFLARE_TTL}"
+    cloudflareagent_log "---"
+    cloudflareagent_log "TLS enabled: ${TLS_ENABLED}"
+    cloudflareagent_log "Letsencrypt enabled: ${LETSENCRYPT_ENABLED}"
+    cloudflareagent_log "Letsencrypt email address: ${LETSENCRYPT_EMAIL}"
+    cloudflareagent_log "---"
+    cloudflareagent_log "Service name: ${SERVICE_NAME}"
+fi
+
+SERVICE_TYPE=$(kubectl get service ${SERVICE_NAME} -o jsonpath='{.spec.type}')
+if [[ "${SERVICE_TYPE}" != "LoadBalancer" ]]; then
+    cloudflareagent_log "Service '${SERVICE_NAME}' is not using 'LoadBalancer', it's using '${SERVICE_TYPE}'"
+    cloudflareagent_log "This process doesn't need to run so is exiting with success"
+    exit 0
+fi
+
+EXTERNAL_IP=""
+while [ -z "${EXTERNAL_IP}" ]; do
+    cloudflareagent_log "Waiting for external IP address for '${SERVICE_NAME}'..."
+    EXTERNAL_IP=$(kubectl get service ${SERVICE_NAME} --template="{{ range .status.loadBalancer.ingress }}{{ .ip }}{{ end }}")
+    [ -z "${EXTERNAL_IP}" ] && sleep 10
+done
+cloudflareagent_log "External IP for '${SERVICE_NAME}' is ready"
+cloudflareagent_log "${EXTERNAL_IP}"
+
+# look up zone ID for provided domain
+ZONE_ID=$(curl -s -H "Content-Type: application/json" -H "X-Auth-Key: ${API_KEY}" -H "X-Auth-Email: ${EMAIL}" -X GET "https://api.cloudflare.com/client/v${API_VERSION}/zones?name=${CLOUDFLARE_DOMAIN}" | jq -r '.result[].id')
+# exit if we can't get it
+if [[ "${ZONE_ID}" == "null" || "${ZONE_ID}" == "" ]]; then
+    cloudflareagent_log "Couldn't get Cloudflare Zone ID for '${CLOUDFLARE_DOMAIN}' with the provided credentials. Exiting"
+    exit 1
+fi
+
+# set TTL if provided - if not, omit it so cloudflare uses auto
+if [[ "${CLOUDFLARE_TTL}" != "" ]]; then
+    RECORD_CONTENT="{\"type\":\"A\",\"name\":\"${DOMAIN_TO_REGISTER}\",\"content\":\"${EXTERNAL_IP}\",\"proxied\":false,\"ttl\":${CLOUDFLARE_TTL}}"
+else
+    RECORD_CONTENT="{\"type\":\"A\",\"name\":\"${DOMAIN_TO_REGISTER}\",\"content\":\"${EXTERNAL_IP}\",\"proxied\":false}"
+fi
+
+# look up record ID
+RECORD_ID=$(curl -s -H "Content-Type: application/json" -H "X-Auth-Key: ${API_KEY}" -H "X-Auth-Email: ${EMAIL}" -X GET "https://api.cloudflare.com/client/v${API_VERSION}/zones/${ZONE_ID}/dns_records?name=${DOMAIN_TO_REGISTER}" | jq -r '.result[].id')
+# if it doesn't exist, create a new record
+if [[ "${RECORD_ID}" == "null" || "${RECORD_ID}" == "" ]]; then
+    cloudflareagent_log "Couldn't get Cloudflare DNS record ID for '${DOMAIN_TO_REGISTER}' within zone '${ZONE_ID}' - creating new record"
+    # create record
+    CREATED_RECORD_ID=$(curl -s -H "Content-Type: application/json" -H "X-Auth-Key: ${API_KEY}" -H "X-Auth-Email: ${EMAIL}" --data ${RECORD_CONTENT} -X POST "https://api.cloudflare.com/client/v${API_VERSION}/zones/${ZONE_ID}/dns_records" | jq -r '.result.id')
+    # check response
+    if [[ "${CREATED_RECORD_ID}" == "null" || "${CREATED_RECORD_ID}" == "" ]]; then
+        cloudflareagent_log "Couldn't create Cloudflare DNS record for '${DOMAIN_TO_REGISTER}' under '${ZONE_ID}'. Exiting"
+        exit 2
+    else
+        cloudflareagent_log "Created Cloudflare DNS record '${CREATED_RECORD_ID}' for '${CLOUDFLARE_DOMAIN}' under '${ZONE_ID}'"
+    fi
+# if it does exist, update the existing record
+else
+    cloudflareagent_log "Got Cloudflare DNS record ID '${RECORD_ID}' for '${DOMAIN_TO_REGISTER}' - updating record"
+    # update record
+    UPDATED_RECORD_ID=$(curl -s -H "Content-Type: application/json" -H "X-Auth-Key: ${API_KEY}" -H "X-Auth-Email: ${EMAIL}" --data ${RECORD_CONTENT} -X PUT "https://api.cloudflare.com/client/v${API_VERSION}/zones/${ZONE_ID}/dns_records/${RECORD_ID}" | jq -r '.result.id')
+    # check response
+    if [[ "${UPDATED_RECORD_ID}" == "null" || "${UPDATED_RECORD_ID}" == "" ]]; then
+        cloudflareagent_log "Couldn't update Cloudflare DNS record for '${DOMAIN_TO_REGISTER}' under '${ZONE_ID}'. Exiting"
+        exit 3
+    else
+        cloudflareagent_log "Updated Cloudflare DNS record '${UPDATED_RECORD_ID}' for '${DOMAIN_TO_REGISTER}' under '${ZONE_ID}'"
+    fi
+fi
+
+# run certbot if TLS is enabled and letsencrypt is enabled
+if [[ "${TLS_ENABLED}" == "true" ]] && [[ "${LETSENCRYPT_ENABLED}" == "true" ]]; then
+    cloudflareagent_log "TLS/Letsencrypt enabled, running certbot"
+    # create certbot.ini file
+    cat >/tmp/cloudflare-credentials-certbot.ini <<EOF
+    dns_cloudflare_email = ${EMAIL}
+    dns_cloudflare_api_key = ${API_KEY}
+EOF
+    chmod 600 /tmp/cloudflare-credentials-certbot.ini
+    certbot certonly -n --agree-tos --email ${LETSENCRYPT_EMAIL} --dns-cloudflare --dns-cloudflare-credentials /tmp/cloudflare-credentials-certbot.ini -d ${DOMAIN_TO_REGISTER}
+    FIRST_TIME=true
+else
+    cloudflareagent_log "TLS/Letsencrypt not enabled, exiting"
+    exit 0
+fi
+
+# keep container running in a loop, attempt to renew certificates once a day and then update kubernetes secrets with changed certificates/key
+while true; do
+    date
+    certbot renew
+    kubectl --namespace ${NAMESPACE} create secret generic tls-web --from-file=/etc/letsencrypt/live/${DOMAIN_TO_REGISTER}/fullchain.pem --from-file=/etc/letsencrypt/live/${DOMAIN_TO_REGISTER}/privkey.pem --dry-run -o yaml | kubectl apply -f -
+    # wait a day
+    sleep 86400
+done

examples/chart/teleport-demo/docker/namespace-cleaner/Dockerfile

@@ -0,0 +1,19 @@
+FROM quay.io/gravitational/debian-grande:latest
+
+ARG KUBECTL_VERSION="v1.12.5"
+ARG CURL_OPTS="-L --retry 100 --retry-delay 0 --connect-timeout 10 --max-time 300"
+
+# Update packages
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && \
+    apt-get -y install curl && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt
+
+# install kubectl
+RUN curl ${CURL_OPTS} https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl --output /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl
+
+COPY rootfs/ /
+
+ENTRYPOINT ["/usr/bin/dumb-init", "/scripts/namespace-cleaner.sh"]

examples/chart/teleport-demo/docker/namespace-cleaner/build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -e
+VERSION=3.2.0
+if [[ "$1" != "" ]]; then
+    VERSION=$1
+    shift
+fi
+docker pull quay.io/gravitational/debian-grande:latest
+docker build --pull \
+    -t gcr.io/kubeadm-167321/namespace-cleaner:${VERSION} \
+    -t gcr.io/kubeadm-167321/namespace-cleaner:latest \
+    --cache-from quay.io/gravitational/debian-grande:latest,gcr.io/kubeadm-167321/namespace-cleaner:latest \
+    . $*
+docker push gcr.io/kubeadm-167321/namespace-cleaner:${VERSION}
+docker push gcr.io/kubeadm-167321/namespace-cleaner:latest

examples/chart/teleport-demo/docker/namespace-cleaner/rootfs/scripts/namespace-cleaner.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# This script takes a space-separated list of namespaces which should be deleted by kubectl after the helm chart is gone
+set -e
+
+function namespacecleaner_log() {
+    echo "[namespace-cleaner] $*"
+}
+
+# handle the case where no arguments are provided
+if [[ $# -eq 0 ]]; then
+    namespacecleaner_log "No namespaces passed as command-line arguments, exiting"
+    exit 0
+# parse arguments on command line
+else
+    namespacecleaner_log "Arguments: '$@'"
+    while [[ $# -gt 0 ]]; do
+        arg="$1"
+        namespacecleaner_log "Processing '${arg}'"
+        # handle colon separated namespace:secret
+        NAMESPACE=${arg}
+        kubectl delete namespace ${NAMESPACE} --wait || true
+        shift
+    done
+fi
+
+namespacecleaner_log "Done - exiting"
+exit 0

examples/chart/teleport-demo/docker/teleport-ent/Dockerfile

@@ -0,0 +1,6 @@
+ARG TELEPORT_VERSION
+FROM quay.io/gravitational/teleport-ent:${TELEPORT_VERSION}
+
+COPY rootfs/ /
+
+ENTRYPOINT ["/usr/bin/dumb-init", "/scripts/teleport.sh"]