more changes to vdc (hail-is#4725)

* wip

* wip

* more tweaks

* option to take letsencrypt from yaml

* tweaks

* tweaks

* wip

* bump

* add gcloud auth in ci test

* set project to broad-ctsa in hail build

* addressed comments

Makefile

@@ -1,6 +1,8 @@
 .PHONY: hail-ci-build-image push-hail-ci-build-image
 .DEFAULT_GOAL := default
 
+PROJECT = $(shell gcloud config get-value project)
+
 # We need to pull each image we use as a cache and we must insure the FROM image
 # is in our cache-from list. I also include the local `hail-pr-builder` to
 # ensure that local iteration on the Dockerfile also uses the cache.
@@ -12,7 +14,7 @@ hail-ci-build-image:
 	    --cache-from $(shell cat hail-ci-build-image),hail-pr-builder,debian:9.5
 
 push-hail-ci-build-image: hail-ci-build-image
-	echo "gcr.io/broad-ctsa/hail-pr-builder:`docker images -q --no-trunc hail-pr-builder:latest | sed -e 's,[^:]*:,,'`" > hail-ci-build-image
+	echo "gcr.io/$(PROJECT)/hail-pr-builder:`docker images -q --no-trunc hail-pr-builder:latest | sed -e 's,[^:]*:,,'`" > hail-ci-build-image
 	docker tag hail-pr-builder `cat hail-ci-build-image`
 	docker push `cat hail-ci-build-image`
 

batch/Makefile

@@ -1,47 +1,43 @@
-.PHONY: build build-batch build-batch-test \
-  push push-batch push-batch-test \
+.PHONY: build build-test \
+  push push-test \
   run-docker run \
-  test test-local deploy-batch
+  test test-local deploy
 
 PROJECT = $(shell gcloud config get-value project)
 
-build: build-batch build-batch-test
-
-build-batch:
+build:
 	docker build -t batch .
 
-build-batch-test:
+build-test:
 	docker build -t batch-test -f Dockerfile.test .
 
-push: push-batch push-batch-test
-
-push-batch: IMAGE="gcr.io/$(PROJECT)/batch:$(shell docker images -q --no-trunc batch | sed -e 's,[^:]*:,,')"
-push-batch: build-batch
+push: IMAGE="gcr.io/$(PROJECT)/batch:$(shell docker images -q --no-trunc batch | sed -e 's,[^:]*:,,')"
+push: build
 	docker tag batch $(IMAGE)
 	docker push $(IMAGE)
 	echo $(IMAGE) > batch-image
 
-push-batch-test: IMAGE="gcr.io/$(PROJECT)/batch-test:$(shell docker images -q --no-trunc batch-test | sed -e 's,[^:]*:,,')"
-push-batch-test: build-batch-test
+push-test: IMAGE="gcr.io/$(PROJECT)/batch-test:$(shell docker images -q --no-trunc batch-test | sed -e 's,[^:]*:,,')"
+push-test: build-test
 	docker tag batch-test $(IMAGE)
 	docker push $(IMAGE)
 	echo $(IMAGE) > batch-test-image
 
-run-docker:
+run-docker: build
 	docker run -e BATCH_USE_KUBE_CONFIG=1 -i -v $(HOME)/.kube:/root/.kube -p 5000:5000 -t batch
 
 run:
 	BATCH_USE_KUBE_CONFIG=1 python batch/server.py
 
-test:
+test: push-test
 	sed -e "s,@image@,$$(cat batch-test-image)," \
 	  < test-batch-pod.yaml.in > test-batch-pod.yaml
 	kubectl create -f test-batch-pod.yaml
 
 test-local:
 	POD_IP='127.0.0.1' BATCH_URL='http://127.0.0.1:5000' python -m unittest -v test/test_batch.py
 
-deploy-batch: push-batch
+deploy: push
 	sed -e "s,@sha@,$$(git rev-parse --short=12 HEAD)," \
 	  -e "s,@image@,$$(cat batch-image)," \
 	  < deployment.yaml.in > deployment.yaml

ci/Makefile

@@ -2,6 +2,8 @@
 .PHONY: setup-conda-env build-hail-ci push-hail-ci test-locally
 .PHONY: run-service deploy
 
+PROJECT = $(shell gcloud config get-value project)
+
 HAIL_CI_LOCAL_BATCH_PORT ?= 8888
 
 setup-conda-env:
@@ -13,7 +15,7 @@ update-conda-env:
 build-hail-ci:
 	cd ../ && docker build . -t hail-ci -f ci/Dockerfile
 
-push-hail-ci: IMAGE = gcr.io/broad-ctsa/hail-ci:$(shell docker images -q --no-trunc hail-ci | head -n 1 | sed -e 's,[^:]*:,,')
+push-hail-ci: IMAGE = gcr.io/$(PROJECT)/hail-ci:$(shell docker images -q --no-trunc hail-ci | head -n 1 | sed -e 's,[^:]*:,,')
 push-hail-ci: build-hail-ci
 	docker tag hail-ci ${IMAGE}
 	docker push ${IMAGE}
@@ -27,7 +29,7 @@ restart-proxy:
 	-kill -9 $(shell cat proxy.pid)
 	-rm -rf proxy.pid
 	$(shell gcloud compute \
-	  --project "broad-ctsa" \
+	  --project "$(PROJECT)" \
 	  ssh \
 	  --zone "us-central1-a" \
 	  "dk-test" \
@@ -74,7 +76,7 @@ test-locally: restart-all-proxies
 	./test-locally.sh
 
 run-service:
-	kubectl apply -f service.yaml
+	kubectl apply -f k8s/service.yaml
 
 deploy: push-hail-ci
 	sed -e "s,@sha@,$(shell git rev-parse --short=12 HEAD)," \

ci/deployment.yaml.in

@@ -42,18 +42,3 @@ spec:
         - name: hail-ci-0-1-github-oauth-token
           secret:
             secretName: hail-ci-0-1-github-oauth-token
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: hail-ci
-  labels:
-    app: hail-ci
-spec:
-  ports:
-    - name: http
-      port: 80
-      protocol: TCP
-      targetPort: 5000
-  selector:
-    app: hail-ci

ci/test-in-cluster.sh

@@ -28,4 +28,6 @@ export IN_CLUSTER=true
 export SELF_HOSTNAME=https://ci.hail.is/$SERVICE_NAME
 export BATCH_SERVER_URL=http://batch.default
 
+gcloud auth activate-service-account --key-file=/secrets/hail-ci-0-1.key
+
 ./test-locally.sh

gateway/.gitignore

@@ -1,3 +1,4 @@
+/Dockerfile
 /deployment.yaml
 /hail.nginx.conf
 /service.yaml

gateway/Dockerfile → gateway/Dockerfile.in

@@ -5,7 +5,7 @@ RUN apt-get update -y && \
     rm -rf /var/lib/apt/lists/*
 
 RUN rm -f /etc/nginx/sites-enabled/default
-ADD hail.nginx.conf /etc/nginx/conf.d/hail.conf
+ADD @nginx_conf@ /etc/nginx/conf.d/hail.conf
 
 RUN ln -sf /dev/stdout /var/log/nginx/access.log
 RUN ln -sf /dev/stderr /var/log/nginx/error.log

gateway/Makefile

@@ -1,11 +1,23 @@
-.PHONY: hail.nginx.conf service.yaml deployment.yaml build push deploy clean
+.PHONY: Dockerfile hail.nginx.conf service.yaml deployment.yaml build push deploy clean
 
 PROJECT = $(shell gcloud config get-value project)
 DOMAIN ?= hail.is
 IP ?= 35.224.105.117
 
+LETSENCRYPT_ONLY = 0
+
+ifeq ($(LETSENCRYPT_ONLY),1)
+NGINX_CONF = letsencrypt.nginx.conf
+else
+NGINX_CONF = hail.nginx.conf
+endif
+
 IMAGE = gcr.io/$(PROJECT)/gateway:$(shell docker images -q --no-trunc gateway | sed -e 's,[^:]*:,,')
 
+Dockerfile: Dockerfile.in
+	sed -e "s,@nginx_conf@,$(NGINX_CONF),g" \
+	  < $< > $@
+
 hail.nginx.conf: hail.nginx.conf.in
 	sed -e "s,@domain@,$(DOMAIN),g" \
 	  < $< > $@
@@ -19,7 +31,7 @@ deployment.yaml: deployment.yaml.in build
 	  -e "s,@image@,$(IMAGE)," \
 	  < $< > $@
 
-build: hail.nginx.conf
+build: Dockerfile $(NGINX_CONF)
 	docker build -t gateway .
 
 push: build

gateway/deployment.yaml.in

@@ -16,6 +16,7 @@ spec:
         app: gateway
         hail.is/sha: "@sha@"
     spec:
+      serviceAccountName: gateway
       containers:
       - name: gateway
         image: @image@

gateway/letsencrypt.nginx.conf

@@ -0,0 +1,18 @@
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+    server_name _;
+
+    location /.well-known/acme-challenge {
+        proxy_pass http://letsencrypt;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+
+    location / {
+        return 404;
+    }
+}

hail/hail-ci-build.sh

@@ -11,6 +11,7 @@ time pip install -U cloudtools
 gcloud auth activate-service-account \
     hail-ci-0-1@broad-ctsa.iam.gserviceaccount.com \
     --key-file=/secrets/hail-ci-0-1.key
+gcloud config set project broad-ctsa
 
 mkdir -p build
 

image-fetcher/Makefile

@@ -1,9 +1,11 @@
 .PHONY: build push deploy
 
+PROJECT = $(shell gcloud config get-value project)
+
 build:
 	docker build . -t image-fetcher
 
-push: IMAGE = gcr.io/broad-ctsa/image-fetcher:$(shell docker images -q --no-trunc image-fetcher | sed -e 's,[^:]*:,,')
+push: IMAGE = gcr.io/$(PROJECT)/image-fetcher:$(shell docker images -q --no-trunc image-fetcher | sed -e 's,[^:]*:,,')
 push: build
 	echo $(IMAGE) > image-fetcher-image
 	docker tag image-fetcher $(IMAGE)

letsencrypt/Makefile

@@ -1,10 +1,9 @@
 STATIC_CONFIG = letsencrypt-pod.yaml letsencrypt.sh
 
-.PHONY: $(STATIC_CONFIG) build push run clean
+.PHONY: $(STATIC_CONFIG) build push start-service run clean
 
 PROJECT = $(shell gcloud config get-value project)
 DOMAIN ?= hail.is
-IP ?= 35.224.105.117
 
 $(STATIC_CONFIG): %: %.in
 	sed -e "s,@project@,$(PROJECT),g" \
@@ -20,6 +19,9 @@ push: build
 	docker tag letsencrypt gcr.io/$(PROJECT)/letsencrypt
 	docker push gcr.io/$(PROJECT)/letsencrypt
 
+start-service: service.yaml
+	kubectl apply -f service.yaml
+
 run: letsencrypt-pod.yaml service.yaml push
 	/bin/bash run-letsencrypt.sh
 

letsencrypt/domains.txt

@@ -1,8 +1,8 @@
 @domain@
-www.@domain@
 ci.@domain@
-upload.@domain@
-scorecard.@domain@
+dev1.@domain@
 notebook.@domain@
+scorecard.@domain@
 test.@domain@
-dev1.@domain@
+upload.@domain@
+www.@domain@

letsencrypt/letsencrypt-pod.yaml.in

@@ -5,6 +5,7 @@ metadata:
   labels:
     app: letsencrypt
 spec:
+  serviceAccountName: letsencrypt
   containers:
   - name: letsencrypt
     image: gcr.io/@project@/letsencrypt

letsencrypt/letsencrypt.sh.in

@@ -7,7 +7,7 @@ sleep 5
 
 rm -rf /etc/letsencrypt/*
 
-# test 
+# test: --test-cert
 certbot --cert-name @domain@ -n --agree-tos -m cseed@broadinstitute.org -d @domains@ --nginx
 
 cat /etc/nginx/conf.d/letsencrypt.conf

notebook/Makefile

@@ -1,9 +1,11 @@
 .PHONY: build push run-docker run deploy clean-workers build-worker push-worker
 
+PROJECT = $(shell gcloud config get-value project)
+
 build: push-worker
 	docker build . -t notebook
 
-push: IMAGE = gcr.io/broad-ctsa/notebook:$(shell docker images -q --no-trunc notebook | sed -e 's,[^:]*:,,')
+push: IMAGE = gcr.io/$(PROJECT)/notebook:$(shell docker images -q --no-trunc notebook | sed -e 's,[^:]*:,,')
 push: build
 	echo $(IMAGE) > notebook-image
 	docker tag notebook $(IMAGE)
@@ -12,7 +14,7 @@ push: build
 build-worker:
 	cd worker && docker build . -t notebook-worker
 
-push-worker: IMAGE = gcr.io/broad-ctsa/notebook-worker:$(shell docker images -q --no-trunc notebook-worker | sed -e 's,[^:]*:,,')
+push-worker: IMAGE = gcr.io/$(PROJECT)/notebook-worker:$(shell docker images -q --no-trunc notebook-worker | sed -e 's,[^:]*:,,')
 push-worker: build-worker
 	echo $(IMAGE) > notebook-worker-image
 	docker tag notebook-worker $(IMAGE)

scorecard/Makefile

@@ -11,7 +11,7 @@ push: build
 	docker tag scorecard $(IMAGE)
 	docker push $(IMAGE)
 
-run-docker:
+run-docker: build
 	docker run -i -p 5000:5000 -v secrets:/secrets -t scorecard
 
 run:

spark/.gitignore

@@ -0,0 +1,2 @@
+/deployment.yaml
+/spark-hail-pod.yaml

spark/Makefile

@@ -1,14 +1,16 @@
 .PHONY: build push run run-hail rm deploy
 
+PROJECT = $(shell gcloud config get-value project)
+
 build:
 	docker build -t spark-base .
 	docker build -t spark-master -f Dockerfile.master .
 	docker build -t spark-worker -f Dockerfile.worker .
 	docker build -t spark-hail -f Dockerfile.hail .
 
-push: SPARK_MASTER_IMAGE="gcr.io/broad-ctsa/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')"
-push: SPARK_WORKER_IMAGE="gcr.io/broad-ctsa/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')"
-push: SPARK_HAIL_IMAGE="gcr.io/broad-ctsa/spark-hail:$(shell docker images -q --no-trunc spark-hail | sed -e 's,[^:]*:,,')"
+push: SPARK_MASTER_IMAGE="gcr.io/$(PROJECT)/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')"
+push: SPARK_WORKER_IMAGE="gcr.io/$(PROJECT)/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')"
+push: SPARK_HAIL_IMAGE="gcr.io/$(PROJECT)/spark-hail:$(shell docker images -q --no-trunc spark-hail | sed -e 's,[^:]*:,,')"
 push: build
 	docker tag spark-master $(SPARK_MASTER_IMAGE)
 	docker push $(SPARK_MASTER_IMAGE)
@@ -35,8 +37,8 @@ run-hail-pod:
 	  < spark-hail-pod.yaml.in > spark-hail-pod.yaml
 	kubectl create -f spark-hail-pod.yaml
 
-deploy: SPARK_MASTER_IMAGE="gcr.io/broad-ctsa/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')"
-deploy: SPARK_WORKER_IMAGE="gcr.io/broad-ctsa/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')"
+deploy: SPARK_MASTER_IMAGE="gcr.io/$(PROJECT)/spark-master:$(shell docker images -q --no-trunc spark-master | sed -e 's,[^:]*:,,')"
+deploy: SPARK_WORKER_IMAGE="gcr.io/$(PROJECT)/spark-worker:$(shell docker images -q --no-trunc spark-worker | sed -e 's,[^:]*:,,')"
 deploy: push
 	sed -e "s,@spark_master_image@,$(SPARK_MASTER_IMAGE),g" \
 	  -e "s,@spark_worker_image@,$(SPARK_WORKER_IMAGE),g" \

vdc/.gitignore

@@ -0,0 +1,5 @@
+/secrets.yaml
+/gcp-config.yaml
+/gcr-pull.json
+/gcr-push-service-account-key.json
+/letsencrypt-config.yaml