From ebe4d6f7cdc99ae3cf2a5f5ee3229f50d7092e1f Mon Sep 17 00:00:00 2001 From: cseed Date: Tue, 6 Nov 2018 14:32:45 -0500 Subject: [PATCH] more changes to vdc (#4725) * wip * wip * more tweaks * option to take letsencrypt from yaml * tweaks * tweaks * wip * bump * add gcloud auth in ci test * set project to broad-ctsa in hail build * addressed comments --- Makefile | 4 +- batch/Makefile | 28 +++--- ci/Makefile | 8 +- ci/deployment.yaml.in | 15 --- ci/test-in-cluster.sh | 2 + gateway/.gitignore | 1 + gateway/{Dockerfile => Dockerfile.in} | 2 +- gateway/Makefile | 16 +++- gateway/deployment.yaml.in | 1 + gateway/letsencrypt.nginx.conf | 18 ++++ hail/hail-ci-build.sh | 1 + image-fetcher/Makefile | 4 +- letsencrypt/Makefile | 6 +- letsencrypt/certs-volume.yaml | 10 -- letsencrypt/domains.txt | 8 +- letsencrypt/letsencrypt-pod.yaml.in | 1 + letsencrypt/letsencrypt.sh.in | 2 +- notebook/Makefile | 6 +- scorecard/Makefile | 2 +- spark/.gitignore | 2 + spark/Makefile | 12 ++- vdc/.gitignore | 5 + vdc/Makefile | 117 ++++++++++++++++++------ vdc/README.md | 33 +++---- vdc/delete-gcr-images.sh | 4 +- vdc/{address.yaml => gcp-address.yaml} | 3 +- vdc/{config.yaml => gcp-config.yaml.in} | 42 +++++---- vdc/k8s-config.yaml | 71 ++++++++++++++ 28 files changed, 290 insertions(+), 134 deletions(-) rename gateway/{Dockerfile => Dockerfile.in} (85%) create mode 100644 gateway/letsencrypt.nginx.conf delete mode 100644 letsencrypt/certs-volume.yaml create mode 100644 spark/.gitignore create mode 100644 vdc/.gitignore rename vdc/{address.yaml => gcp-address.yaml} (73%) rename vdc/{config.yaml => gcp-config.yaml.in} (77%) diff --git a/Makefile b/Makefile index 081b310155f..1291b8bd5fa 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ .PHONY: hail-ci-build-image push-hail-ci-build-image .DEFAULT_GOAL := default +PROJECT = $(shell gcloud config get-value project) + # We need to pull each image we use as a cache and we must insure the FROM image # is in our cache-from list. I also include the local `hail-pr-builder` to # ensure that local iteration on the Dockerfile also uses the cache. @@ -12,7 +14,7 @@ hail-ci-build-image: --cache-from $(shell cat hail-ci-build-image),hail-pr-builder,debian:9.5 push-hail-ci-build-image: hail-ci-build-image - echo "gcr.io/broad-ctsa/hail-pr-builder:`docker images -q --no-trunc hail-pr-builder:latest | sed -e 's,[^:]*:,,'`" > hail-ci-build-image + echo "gcr.io/$(PROJECT)/hail-pr-builder:`docker images -q --no-trunc hail-pr-builder:latest | sed -e 's,[^:]*:,,'`" > hail-ci-build-image docker tag hail-pr-builder `cat hail-ci-build-image` docker push `cat hail-ci-build-image` diff --git a/batch/Makefile b/batch/Makefile index 7c5d81b8c55..f20c1cce92d 100644 --- a/batch/Makefile +++ b/batch/Makefile @@ -1,39 +1,35 @@ -.PHONY: build build-batch build-batch-test \ - push push-batch push-batch-test \ +.PHONY: build build-test \ + push push-test \ run-docker run \ - test test-local deploy-batch + test test-local deploy PROJECT = $(shell gcloud config get-value project) -build: build-batch build-batch-test - -build-batch: +build: docker build -t batch . -build-batch-test: +build-test: docker build -t batch-test -f Dockerfile.test . -push: push-batch push-batch-test - -push-batch: IMAGE="gcr.io/$(PROJECT)/batch:$(shell docker images -q --no-trunc batch | sed -e 's,[^:]*:,,')" -push-batch: build-batch +push: IMAGE="gcr.io/$(PROJECT)/batch:$(shell docker images -q --no-trunc batch | sed -e 's,[^:]*:,,')" +push: build docker tag batch $(IMAGE) docker push $(IMAGE) echo $(IMAGE) > batch-image -push-batch-test: IMAGE="gcr.io/$(PROJECT)/batch-test:$(shell docker images -q --no-trunc batch-test | sed -e 's,[^:]*:,,')" -push-batch-test: build-batch-test +push-test: IMAGE="gcr.io/$(PROJECT)/batch-test:$(shell docker images -q --no-trunc batch-test | sed -e 's,[^:]*:,,')" +push-test: build-test docker tag batch-test $(IMAGE) docker push $(IMAGE) echo $(IMAGE) > batch-test-image -run-docker: +run-docker: build docker run -e BATCH_USE_KUBE_CONFIG=1 -i -v $(HOME)/.kube:/root/.kube -p 5000:5000 -t batch run: BATCH_USE_KUBE_CONFIG=1 python batch/server.py -test: +test: push-test sed -e "s,@image@,$$(cat batch-test-image)," \ < test-batch-pod.yaml.in > test-batch-pod.yaml kubectl create -f test-batch-pod.yaml @@ -41,7 +37,7 @@ test: test-local: POD_IP='127.0.0.1' BATCH_URL='http://127.0.0.1:5000' python -m unittest -v test/test_batch.py -deploy-batch: push-batch +deploy: push sed -e "s,@sha@,$$(git rev-parse --short=12 HEAD)," \ -e "s,@image@,$$(cat batch-image)," \ < deployment.yaml.in > deployment.yaml diff --git a/ci/Makefile b/ci/Makefile index bffe112f7de..ce72d7bc760 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -2,6 +2,8 @@ .PHONY: setup-conda-env build-hail-ci push-hail-ci test-locally .PHONY: run-service deploy +PROJECT = $(shell gcloud config get-value project) + HAIL_CI_LOCAL_BATCH_PORT ?= 8888 setup-conda-env: @@ -13,7 +15,7 @@ update-conda-env: build-hail-ci: cd ../ && docker build . -t hail-ci -f ci/Dockerfile -push-hail-ci: IMAGE = gcr.io/broad-ctsa/hail-ci:$(shell docker images -q --no-trunc hail-ci | head -n 1 | sed -e 's,[^:]*:,,') +push-hail-ci: IMAGE = gcr.io/$(PROJECT)/hail-ci:$(shell docker images -q --no-trunc hail-ci | head -n 1 | sed -e 's,[^:]*:,,') push-hail-ci: build-hail-ci docker tag hail-ci ${IMAGE} docker push ${IMAGE} @@ -27,7 +29,7 @@ restart-proxy: -kill -9 $(shell cat proxy.pid) -rm -rf proxy.pid $(shell gcloud compute \ - --project "broad-ctsa" \ + --project "$(PROJECT)" \ ssh \ --zone "us-central1-a" \ "dk-test" \ @@ -74,7 +76,7 @@ test-locally: restart-all-proxies ./test-locally.sh run-service: - kubectl apply -f service.yaml + kubectl apply -f k8s/service.yaml deploy: push-hail-ci sed -e "s,@sha@,$(shell git rev-parse --short=12 HEAD)," \ diff --git a/ci/deployment.yaml.in b/ci/deployment.yaml.in index 6c85978a70d..2de56fa45f7 100644 --- a/ci/deployment.yaml.in +++ b/ci/deployment.yaml.in @@ -42,18 +42,3 @@ spec: - name: hail-ci-0-1-github-oauth-token secret: secretName: hail-ci-0-1-github-oauth-token ---- -apiVersion: v1 -kind: Service -metadata: - name: hail-ci - labels: - app: hail-ci -spec: - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 5000 - selector: - app: hail-ci diff --git a/ci/test-in-cluster.sh b/ci/test-in-cluster.sh index 44e85a78946..c2f59544b6b 100755 --- a/ci/test-in-cluster.sh +++ b/ci/test-in-cluster.sh @@ -28,4 +28,6 @@ export IN_CLUSTER=true export SELF_HOSTNAME=https://ci.hail.is/$SERVICE_NAME export BATCH_SERVER_URL=http://batch.default +gcloud auth activate-service-account --key-file=/secrets/hail-ci-0-1.key + ./test-locally.sh diff --git a/gateway/.gitignore b/gateway/.gitignore index 07315ed8f3c..f4cf7595633 100644 --- a/gateway/.gitignore +++ b/gateway/.gitignore @@ -1,3 +1,4 @@ +/Dockerfile /deployment.yaml /hail.nginx.conf /service.yaml diff --git a/gateway/Dockerfile b/gateway/Dockerfile.in similarity index 85% rename from gateway/Dockerfile rename to gateway/Dockerfile.in index 2fd4a83f012..40346571587 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile.in @@ -5,7 +5,7 @@ RUN apt-get update -y && \ rm -rf /var/lib/apt/lists/* RUN rm -f /etc/nginx/sites-enabled/default -ADD hail.nginx.conf /etc/nginx/conf.d/hail.conf +ADD @nginx_conf@ /etc/nginx/conf.d/hail.conf RUN ln -sf /dev/stdout /var/log/nginx/access.log RUN ln -sf /dev/stderr /var/log/nginx/error.log diff --git a/gateway/Makefile b/gateway/Makefile index 80e8f510015..c426676e62b 100644 --- a/gateway/Makefile +++ b/gateway/Makefile @@ -1,11 +1,23 @@ -.PHONY: hail.nginx.conf service.yaml deployment.yaml build push deploy clean +.PHONY: Dockerfile hail.nginx.conf service.yaml deployment.yaml build push deploy clean PROJECT = $(shell gcloud config get-value project) DOMAIN ?= hail.is IP ?= 35.224.105.117 +LETSENCRYPT_ONLY = 0 + +ifeq ($(LETSENCRYPT_ONLY),1) +NGINX_CONF = letsencrypt.nginx.conf +else +NGINX_CONF = hail.nginx.conf +endif + IMAGE = gcr.io/$(PROJECT)/gateway:$(shell docker images -q --no-trunc gateway | sed -e 's,[^:]*:,,') +Dockerfile: Dockerfile.in + sed -e "s,@nginx_conf@,$(NGINX_CONF),g" \ + < $< > $@ + hail.nginx.conf: hail.nginx.conf.in sed -e "s,@domain@,$(DOMAIN),g" \ < $< > $@ @@ -19,7 +31,7 @@ deployment.yaml: deployment.yaml.in build -e "s,@image@,$(IMAGE)," \ < $< > $@ -build: hail.nginx.conf +build: Dockerfile $(NGINX_CONF) docker build -t gateway . push: build diff --git a/gateway/deployment.yaml.in b/gateway/deployment.yaml.in index 4a734a71094..d3f3cc034a5 100644 --- a/gateway/deployment.yaml.in +++ b/gateway/deployment.yaml.in @@ -16,6 +16,7 @@ spec: app: gateway hail.is/sha: "@sha@" spec: + serviceAccountName: gateway containers: - name: gateway image: @image@ diff --git a/gateway/letsencrypt.nginx.conf b/gateway/letsencrypt.nginx.conf new file mode 100644 index 00000000000..22c22314c31 --- /dev/null +++ b/gateway/letsencrypt.nginx.conf @@ -0,0 +1,18 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + location /.well-known/acme-challenge { + proxy_pass http://letsencrypt; + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto https; + } + + location / { + return 404; + } +} diff --git a/hail/hail-ci-build.sh b/hail/hail-ci-build.sh index 35111de77f3..89a69905987 100644 --- a/hail/hail-ci-build.sh +++ b/hail/hail-ci-build.sh @@ -11,6 +11,7 @@ time pip install -U cloudtools gcloud auth activate-service-account \ hail-ci-0-1@broad-ctsa.iam.gserviceaccount.com \ --key-file=/secrets/hail-ci-0-1.key +gcloud config set project broad-ctsa mkdir -p build diff --git a/image-fetcher/Makefile b/image-fetcher/Makefile index 737e6a90959..0df9c97c40c 100644 --- a/image-fetcher/Makefile +++ b/image-fetcher/Makefile @@ -1,9 +1,11 @@ .PHONY: build push deploy +PROJECT = $(shell gcloud config get-value project) + build: docker build . -t image-fetcher -push: IMAGE = gcr.io/broad-ctsa/image-fetcher:$(shell docker images -q --no-trunc image-fetcher | sed -e 's,[^:]*:,,') +push: IMAGE = gcr.io/$(PROJECT)/image-fetcher:$(shell docker images -q --no-trunc image-fetcher | sed -e 's,[^:]*:,,') push: build echo $(IMAGE) > image-fetcher-image docker tag image-fetcher $(IMAGE) diff --git a/letsencrypt/Makefile b/letsencrypt/Makefile index bdc80c88fcb..22ad23e649b 100644 --- a/letsencrypt/Makefile +++ b/letsencrypt/Makefile @@ -1,10 +1,9 @@ STATIC_CONFIG = letsencrypt-pod.yaml letsencrypt.sh -.PHONY: $(STATIC_CONFIG) build push run clean +.PHONY: $(STATIC_CONFIG) build push start-service run clean PROJECT = $(shell gcloud config get-value project) DOMAIN ?= hail.is -IP ?= 35.224.105.117 $(STATIC_CONFIG): %: %.in sed -e "s,@project@,$(PROJECT),g" \ @@ -20,6 +19,9 @@ push: build docker tag letsencrypt gcr.io/$(PROJECT)/letsencrypt docker push gcr.io/$(PROJECT)/letsencrypt +start-service: service.yaml + kubectl apply -f service.yaml + run: letsencrypt-pod.yaml service.yaml push /bin/bash run-letsencrypt.sh diff --git a/letsencrypt/certs-volume.yaml b/letsencrypt/certs-volume.yaml deleted file mode 100644 index fa767a9d601..00000000000 --- a/letsencrypt/certs-volume.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: letsencrypt-certs -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Mi diff --git a/letsencrypt/domains.txt b/letsencrypt/domains.txt index 654566f6518..cb0777a6d1f 100644 --- a/letsencrypt/domains.txt +++ b/letsencrypt/domains.txt @@ -1,8 +1,8 @@ @domain@ -www.@domain@ ci.@domain@ -upload.@domain@ -scorecard.@domain@ +dev1.@domain@ notebook.@domain@ +scorecard.@domain@ test.@domain@ -dev1.@domain@ +upload.@domain@ +www.@domain@ diff --git a/letsencrypt/letsencrypt-pod.yaml.in b/letsencrypt/letsencrypt-pod.yaml.in index bc904e2ff90..16a4ddfc38a 100644 --- a/letsencrypt/letsencrypt-pod.yaml.in +++ b/letsencrypt/letsencrypt-pod.yaml.in @@ -5,6 +5,7 @@ metadata: labels: app: letsencrypt spec: + serviceAccountName: letsencrypt containers: - name: letsencrypt image: gcr.io/@project@/letsencrypt diff --git a/letsencrypt/letsencrypt.sh.in b/letsencrypt/letsencrypt.sh.in index ff8a6595de3..62e9866c74d 100644 --- a/letsencrypt/letsencrypt.sh.in +++ b/letsencrypt/letsencrypt.sh.in @@ -7,7 +7,7 @@ sleep 5 rm -rf /etc/letsencrypt/* -# test +# test: --test-cert certbot --cert-name @domain@ -n --agree-tos -m cseed@broadinstitute.org -d @domains@ --nginx cat /etc/nginx/conf.d/letsencrypt.conf diff --git a/notebook/Makefile b/notebook/Makefile index e1ed9bcb2c3..e0290bc8455 100644 --- a/notebook/Makefile +++ b/notebook/Makefile @@ -1,9 +1,11 @@ .PHONY: build push run-docker run deploy clean-workers build-worker push-worker +PROJECT = $(shell gcloud config get-value project) + build: push-worker docker build . -t notebook -push: IMAGE = gcr.io/broad-ctsa/notebook:$(shell docker images -q --no-trunc notebook | sed -e 's,[^:]*:,,') +push: IMAGE = gcr.io/$(PROJECT)/notebook:$(shell docker images -q --no-trunc notebook | sed -e 's,[^:]*:,,') push: build echo $(IMAGE) > notebook-image docker tag notebook $(IMAGE) @@ -12,7 +14,7 @@ push: build build-worker: cd worker && docker build . -t notebook-worker -push-worker: IMAGE = gcr.io/broad-ctsa/notebook-worker:$(shell docker images -q --no-trunc notebook-worker | sed -e 's,[^:]*:,,') +push-worker: IMAGE = gcr.io/$(PROJECT)/notebook-worker:$(shell docker images -q --no-trunc notebook-worker | sed -e 's,[^:]*:,,') push-worker: build-worker echo $(IMAGE) > notebook-worker-image docker tag notebook-worker $(IMAGE) diff --git a/scorecard/Makefile b/scorecard/Makefile index 877b943b5e3..ece5d21bb04 100644 --- a/scorecard/Makefile +++ b/scorecard/Makefile @@ -11,7 +11,7 @@ push: build docker tag scorecard $(IMAGE) docker push $(IMAGE) -run-docker: +run-docker: build docker run -i -p 5000:5000 -v secrets:/secrets -t scorecard run: diff --git a/spark/.gitignore b/spark/.gitignore new file mode 100644 index 00000000000..fc905e76b8d --- /dev/null +++ b/spark/.gitignore @@ -0,0 +1,2 @@ +/deployment.yaml +/spark-hail-pod.yaml diff --git a/spark/Makefile b/spark/Makefile index cb66ba3d65c..46e2fb3c032 100644 --- a/spark/Makefile +++ b/spark/Makefile @@ -1,14 +1,16 @@ .PHONY: build push run run-hail rm deploy +PROJECT = $(shell gcloud config get-value project) + build: docker build -t spark-base . docker build -t spark-master -f Dockerfile.master . docker build -t spark-worker -f Dockerfile.worker . docker build -t spark-hail -f Dockerfile.hail . -push: SPARK_MASTER_IMAGE="gcr.io/broad-ctsa/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')" -push: SPARK_WORKER_IMAGE="gcr.io/broad-ctsa/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')" -push: SPARK_HAIL_IMAGE="gcr.io/broad-ctsa/spark-hail:$(shell docker images -q --no-trunc spark-hail | sed -e 's,[^:]*:,,')" +push: SPARK_MASTER_IMAGE="gcr.io/$(PROJECT)/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')" +push: SPARK_WORKER_IMAGE="gcr.io/$(PROJECT)/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')" +push: SPARK_HAIL_IMAGE="gcr.io/$(PROJECT)/spark-hail:$(shell docker images -q --no-trunc spark-hail | sed -e 's,[^:]*:,,')" push: build docker tag spark-master $(SPARK_MASTER_IMAGE) docker push $(SPARK_MASTER_IMAGE) @@ -35,8 +37,8 @@ run-hail-pod: < spark-hail-pod.yaml.in > spark-hail-pod.yaml kubectl create -f spark-hail-pod.yaml -deploy: SPARK_MASTER_IMAGE="gcr.io/broad-ctsa/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')" -deploy: SPARK_WORKER_IMAGE="gcr.io/broad-ctsa/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')" +deploy: SPARK_MASTER_IMAGE="gcr.io/$(PROJECT)/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')" +deploy: SPARK_WORKER_IMAGE="gcr.io/$(PROJECT)/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')" deploy: push sed -e "s,@spark_master_image@,$(SPARK_MASTER_IMAGE),g" \ -e "s,@spark_worker_image@,$(SPARK_WORKER_IMAGE),g" \ diff --git a/vdc/.gitignore b/vdc/.gitignore new file mode 100644 index 00000000000..9c703d69f45 --- /dev/null +++ b/vdc/.gitignore @@ -0,0 +1,5 @@ +/secrets.yaml +/gcp-config.yaml +/gcr-pull.json +/gcr-push-service-account-key.json +/letsencrypt-config.yaml diff --git a/vdc/Makefile b/vdc/Makefile index a2903d01292..7cd64a3959b 100644 --- a/vdc/Makefile +++ b/vdc/Makefile @@ -1,51 +1,108 @@ -.PHONY: build-out create-deployment build-out-k8s create-k8s-secrets create-k8s-certs-volume deploy-k8s \ - delete-deployment tear-down +.PHONY: echo-ip gcloud-config build-out create-deployment build-out-k8s create-k8s-secrets create-k8s-certs-volume deploy-k8s \ + delete-deployment tear-down create-address -export PROJECT ?= hail-vdc +export PROJECT ?= hail-vdc-staging +export REGION ?= us-central1 +export ZONE ?= $(REGION)-a export DOMAIN ?= staging.hail.is -export IP ?= 35.188.91.25 +export IP ?= $(shell gcloud --project $(PROJECT) compute addresses describe --region $(REGION) site --format='value(address)') + +# run letsencrypt, or take from letsencrypt-config.yaml +RUN_LETSENCRYPT = 1 build-out: create-deployment build-out-k8s -create-deployment: +echo-ip: + echo IP=$(IP) + +activate-deploy: + gcloud iam service-accounts keys create deploy-sa-key.json \ + --iam-account deploy@$(PROJECT).iam.gserviceaccount.com + gcloud auth activate-service-account --key-file=deploy-sa-key.json + rm deploy-sa-key.json + +gcloud-config: + gcloud config set account deploy@$(PROJECT).iam.gserviceaccount.com gcloud config set project $(PROJECT) - gcloud beta -q deployment-manager deployments create default --config config.yaml -# FIXME integrate into config - gsutil iam ch serviceAccount:vdc-svc@$(PROJECT).iam.gserviceaccount.com:objectViewer gs://artifacts.$(PROJECT).appspot.com + gcloud config set compute/region $(REGION) + gcloud config set compute/zone $(ZONE) gcloud container clusters get-credentials vdc - kubectl create clusterrolebinding project-editor-cluster-admin-binding --clusterrole cluster-admin --user projectEditor:$(PROJECT) -build-out-k8s: k8s-config create-k8s-secrets create-k8s-certs-volume run-letsencrypt deploy-k8s +# deployment manager cannot manage appspot.com bucket because it requires domain ownership verification +create-artifacts-bucket: + docker pull alpine:3.8 + docker tag alpine:3.8 gcr.io/$(PROJECT)/alpine:3.8 + docker push gcr.io/$(PROJECT)/alpine:3.8 + +create-deployment: gcloud-config create-artifacts-bucket + sed -e 's,@project@,$(PROJECT),g' \ + -e 's,@region@,$(REGION),g' \ + -e 's,@zone@,$(ZONE),g' \ + < gcp-config.yaml.in > gcp-config.yaml + gcloud beta -q deployment-manager deployments create default --config gcp-config.yaml + kubectl create clusterrolebinding deploy-cluster-admin-binding --clusterrole cluster-admin --user deploy@$(PROJECT).iam.gserviceaccount.com +# FIXME still trying to figure out out to add this to config + gsutil iam ch serviceAccount:gcr-push@$(PROJECT).iam.gserviceaccount.com:admin gs://artifacts.$(PROJECT).appspot.com + gsutil iam ch serviceAccount:gcr-pull@$(PROJECT).iam.gserviceaccount.com:objectViewer gs://artifacts.$(PROJECT).appspot.com + gsutil iam ch serviceAccount:vdc-sa@$(PROJECT).iam.gserviceaccount.com:objectViewer gs://artifacts.$(PROJECT).appspot.com + +build-out-k8s: k8s-config create-k8s-secrets run-letsencrypt deploy-k8s k8s-config: kubectl apply -f k8s-config.yaml create-k8s-secrets: kubectl apply -f secrets.yaml - kubectl create secret generic upload-oauth2-client-secret \ - --from-file=./client_secret.json - -create-k8s-certs-volume: - kubectl apply -f ../site/certs-volume.yaml + kubectl delete secrets --ignore-not-found=true gcr-push-service-account-key gcr-pull-key letsencrypt-config + gcloud iam service-accounts keys create \ + --iam-account=gcr-push@$(PROJECT).iam.gserviceaccount.com \ + gcr-push-service-account-key.json + kubectl create secret generic gcr-push-service-account-key --from-file=gcr-push-service-account-key.json + rm -f gcr-push-service-account-key.json + gcloud iam service-accounts keys create \ + --iam-account=gcr-pull@$(PROJECT).iam.gserviceaccount.com \ + gcr-pull.json + kubectl create secret generic gcr-pull-key --from-file=gcr-pull.json + rm -f gcr-pull.json +# empty secret to be filled in by letsencrypt + kubectl create secret generic letsencrypt-config +ifeq ($(RUN_LETSENCRYPT),1) run-letsencrypt: - $(MAKE) -C ../site push-run-letsencrypt run-letsencrypt + $(MAKE) -C ../letsencrypt start-service + $(MAKE) -C ../gateway LETSENCRYPT_ONLY=1 deploy + kubectl -n default rollout status -w deployment gateway-deployment + $(MAKE) -C ../letsencrypt run +else +run-letsencrypt: + kubectl apply -f letsencrypt-config.yaml +endif deploy-k8s: - $(MAKE) -C ../batch push-batch deploy-batch -# FIXME skip ci until it can handle running as a secondary + $(MAKE) -C ../letsencrypt start-service + $(MAKE) -C ../batch deploy +# FIXME ci can't run as a secondary yet $(MAKE) -C ../ci run-service - $(MAKE) -C ../scorecard push deploy - $(MAKE) -C ../upload push deploy - $(MAKE) -C ../site push-site deploy-site + $(MAKE) -C ../notebook deploy + $(MAKE) -C ../image-fetcher deploy + $(MAKE) -C ../scorecard deploy + $(MAKE) -C ../site deploy + $(MAKE) -C ../upload deploy +# last so the services are up + $(MAKE) -C ../gateway deploy -delete-deployment: - gcloud beta -q --project $(PROJECT) deployment-manager deployments delete default +clean-gcr: + bash delete-gcr-images.sh +# don't fail if doesn't exist + -gsutil -m rm -r gs://artifacts.$(PROJECT).appspot.com -tear-down: -# just to be safe - gcloud config set project $(PROJECT) -# FIXME integrate into config - gsutil iam ch -d serviceAccount:vdc-svc@$(PROJECT).iam.gserviceaccount.com:objectViewer gs://artifacts.$(PROJECT).appspot.com - gcloud beta -q --project $(PROJECT) deployment-manager deployments delete default - /bin/bash delete-gcr-images.sh +tear-down: delete-deployment clean-gcr + +delete-deployment: gcloud-config + gcloud beta -q deployment-manager deployments delete default + +create-address: gcloud-config + gcloud beta -q deployment-manager deployments create address --config gcp-address.yaml + +delete-address: gcloud-config + gcloud beta -q deployment-manager deployments delete address diff --git a/vdc/README.md b/vdc/README.md index 3ba19473678..0d5ef4b0187 100644 --- a/vdc/README.md +++ b/vdc/README.md @@ -10,42 +10,37 @@ beforehand: - Enable Identity and Access Management (IAM) API - Enable Cloud SQL Admin API + - Google Cloud Deployment Manager V2 API - - In the GCP console, go to VPC network > External IP addresses > - RESERVE STATIC IP ADDRESS and reserve a static IP address called - `site`. Also see [address.yaml](address.yaml). + - Reserve a static IP address `site` by running `make create-address`. - Update the domain's DNS to point to `site`'s external IP address. + You can print the IP address by running `make echo-ip`. - - In the GCP console, go to APIs & Services > Credentials > OAuth - consent screen, and configure the consent screen. + - Create a service account + `deploy@.iam.gserviceaccount.com` with the project + owner role. - - In the GCP console, go to APIs & Services > Credentials > - Credentials and create an OAuth client ID. - - - Choose application type "Web application". - - Authorized redirect URIs will be - `https://upload.hail.is/oauth2callback` adjusted for domain. + - Activate the deploy service account in `gcloud` by running `make + activate-deploy`. ### Deploy - - Put the OAuth client ID JSON in `./client_secret.json`. It can be - downloaded from APIs & Services > Credentials > Credentials. - - Put secrets.yaml in `./secrets.yaml`. - Run, for example: ``` -make build-out PROJECT=hail-vdc IP=35.188.91.25 DOMAIN=staging.hail.is +make PROJECT=hail-vdc IP=35.188.91.25 DOMAIN=hail.is build-out ``` - Warning: modifies gcloud project configuration setting + Warning: modifies gcloud, kubectl configuration setting + + - Add `vdc-sa@.iam.gserviceaccount.com` service account + to broad-ctsa/artifacts.broad-ctsa.appspot.com to Storage Object + Viewer role. ### FIXME - Doesn't deploy ci, which can't have multiple running instances. - - Batch likely doesn't work, needs k8s service account to create pods. - Describe secrets.yaml. - - List of APIs to enable is not exhaustive. Run through in a fresh - project. diff --git a/vdc/delete-gcr-images.sh b/vdc/delete-gcr-images.sh index 5dbd6a1f9ad..f567281af2d 100644 --- a/vdc/delete-gcr-images.sh +++ b/vdc/delete-gcr-images.sh @@ -4,8 +4,8 @@ set -ex PROJECT=$(gcloud config get-value project) echo PROJECT=${PROJECT} -for BASE in batch run-letsencrypt scorecard site upload; do - IMAGE=gcr.io/${PROJECT}/${BASE} +for IMAGE in $(gcloud container images list --format 'get(name)'); do + echo IMAGE=${IMAGE} for DIGEST in $(gcloud container images list-tags ${IMAGE} --format 'get(digest)'); do gcloud container images delete -q --force-delete-tags ${IMAGE}@${DIGEST} done diff --git a/vdc/address.yaml b/vdc/gcp-address.yaml similarity index 73% rename from vdc/address.yaml rename to vdc/gcp-address.yaml index f2792b130ab..61a17e9cd44 100644 --- a/vdc/address.yaml +++ b/vdc/gcp-address.yaml @@ -1,6 +1,7 @@ resources: # address -- name: site-address +# name is not taken from property +- name: site type: compute.beta.address properties: name: site diff --git a/vdc/config.yaml b/vdc/gcp-config.yaml.in similarity index 77% rename from vdc/config.yaml rename to vdc/gcp-config.yaml.in index d2fff3d3bb9..e488083cae5 100644 --- a/vdc/config.yaml +++ b/vdc/gcp-config.yaml.in @@ -1,33 +1,39 @@ resources: # service accounts -- name: vdc-service-account +- name: vdc-sa type: iam.v1.serviceAccount properties: - accountId: vdc-svc - displayName: for vdc-svc +# must be 6-30 characters + accountId: vdc-sa + displayName: for vdc - name: gcr-push type: iam.v1.serviceAccount properties: accountId: gcr-push displayName: push to gcr.io -- name: gcr-push-key - type: iam.v1.serviceAccounts.key - metadata: - dependsOn: - - gcr-push +- name: gcr-pull + type: iam.v1.serviceAccount + properties: + accountId: gcr-pull + displayName: pull from gcr.io +- name: k8s-admin + type: iam.v1.serviceAccount properties: - parent: projects/hail-vdc/serviceAccounts/gcr-push@hail-vdc.iam.gserviceaccount.com + accountId: k8s-admin + displayName: k8s admin +# buckets +# deployment manager cannot create artifacts.@project@.appspot.com # sql instance - name: sql-instance type: sqladmin.v1beta4.instance properties: - name: db-07 + name: db-gh0um region: us-central1 settings: activationPolicy: ALWAYS ipConfiguration: ipv4Enabled: false - privateNetwork: projects/hail-vdc/global/networks/default + privateNetwork: projects/@project@/global/networks/default backupConfiguration: enabled: true startTime: 02:00 @@ -41,23 +47,23 @@ resources: type: container.v1.cluster metadata: dependsOn: - - vdc-service-account + - vdc-sa properties: - zone: us-central1-a + zone: @zone@ cluster: addonsConfig: httpLoadBalancing: {} kubernetesDashboard: {disabled: true} initialClusterVersion: 1.10.7-gke.6 ipAllocationPolicy: {useIpAliases: true} - location: us-central1-a + location: @zone@ loggingService: logging.googleapis.com masterAuth: clientCertificateConfig: {} masterAuthorizedNetworksConfig: {} monitoringService: monitoring.googleapis.com name: vdc - network: projects/hail-vdc/global/networks/default + network: projects/@project@/global/networks/default networkPolicy: {} nodePools: - autoscaling: {} @@ -65,7 +71,7 @@ resources: diskSizeGb: 100 diskType: pd-standard imageType: COS - serviceAccount: vdc-svc@hail-vdc.iam.gserviceaccount.com + serviceAccount: vdc-sa@@project@.iam.gserviceaccount.com machineType: n1-standard-1 oauthScopes: ['https://www.googleapis.com/auth/devstorage.read_only', 'https://www.googleapis.com/auth/logging.write', 'https://www.googleapis.com/auth/monitoring', 'https://www.googleapis.com/auth/servicecontrol', @@ -80,7 +86,7 @@ resources: diskType: pd-standard imageType: COS labels: {preemptible: 'true'} - serviceAccount: vdc-svc@hail-vdc.iam.gserviceaccount.com + serviceAccount: vdc-sa@@project@.iam.gserviceaccount.com machineType: n1-standard-8 oauthScopes: ['https://www.googleapis.com/auth/devstorage.read_only', 'https://www.googleapis.com/auth/logging.write', 'https://www.googleapis.com/auth/monitoring', 'https://www.googleapis.com/auth/servicecontrol', @@ -92,4 +98,4 @@ resources: name: preemptible-pool version: 1.10.7-gke.6 privateClusterConfig: {} - subnetwork: projects/hail-vdc/regions/us-central1/subnetworks/default + subnetwork: projects/@project@/regions/@region@/subnetworks/default diff --git a/vdc/k8s-config.yaml b/vdc/k8s-config.yaml index bf410378d9d..ab75f3ac64d 100644 --- a/vdc/k8s-config.yaml +++ b/vdc/k8s-config.yaml @@ -13,6 +13,66 @@ kind: ServiceAccount metadata: name: batch-svc --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gateway +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: letsencrypt +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: read-letsencrypt-config +rules: +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["letsencrypt-config"] + verbs: ["get"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: gateway-read-letsencrypt-config +subjects: +- kind: ServiceAccount + name: gateway + namespace: default +roleRef: + kind: Role + name: read-letsencrypt-config + apiGroup: "" +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: update-letsencrypt-config +rules: +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["letsencrypt-config"] + verbs: ["get", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: letsencrypt-update-letsencrypt-config +subjects: +- kind: ServiceAccount + name: letsencrypt + namespace: default +roleRef: + kind: Role + name: update-letsencrypt-config + apiGroup: "" +--- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -144,3 +204,14 @@ roleRef: kind: Role name: test-test apiGroup: "rbac.authorization.k8s.io" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: letsencrypt-certs +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi