RS-574: Deploy dev infra to the PR development cluster

.github/workflows/PR.yaml

@@ -10,6 +10,9 @@ defaults:
   run:
     working-directory: go/src/github.com/stackrox/infra
 
+env:
+  CLUSTER_NAME: infra-pr-${{ github.event.pull_request.number }}
+
 jobs:
 
   lint:
@@ -31,7 +34,7 @@ jobs:
           wait: true
           token: ${{ secrets.INFRA_TOKEN }}
 
-  comment-for-dev-deploy:
+  comment-on-PR:
     needs:
       - build-and-push
       - create-dev-cluster
@@ -51,4 +54,71 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.ROBOT_ROX_GITHUB_TOKEN }}
         run: |
-          ./scripts/add-PR-comment-for-deploy-to-dev.sh "${{ github.event.pull_request.html_url }}" "infra-pr-${{ github.event.pull_request.number }}"
+          ./scripts/add-PR-comment-for-deploy-to-dev.sh "${{ github.event.pull_request.html_url }}" "$CLUSTER_NAME"
+
+  deploy-to-dev-cluster:
+    needs:
+      - build-and-push
+      - create-dev-cluster
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.3.46
+    env:
+      INFRA_TOKEN: ${{secrets.INFRA_TOKEN}}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: go/src/github.com/stackrox/infra
+
+      - name: Download production infractl
+        run: |
+          mkdir -p bin
+          curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
+          | jq -r ".result.fileChunk" \
+          | base64 -d \
+          > bin/production-infractl
+          chmod +x bin/production-infractl
+
+      - name: Download artifacts
+        run: |
+          mkdir artifacts
+          bin/production-infractl artifacts "$CLUSTER_NAME" -d artifacts
+
+      - name: Deploy infra to dev cluster and check the deployment
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GCP_INFRA_CI_AUTOMATION_SA}}
+          KUBECONFIG: artifacts/kubeconfig
+        run: |
+          gcloud auth activate-service-account --key-file <(echo "$GOOGLE_APPLICATION_CREDENTIALS")
+          gcloud auth list
+
+          make install-local-without-write
+
+          nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 &
+          sleep 5
+          make pull-infractl-from-dev-server
+
+          version="$(bin/infractl -k -e localhost:8443 version --json)"
+          echo "$version"
+          client="$(echo $version | jq -r '.Client.Version')"
+          server="$(echo $version | jq -r '.Server.Version')"
+          if [[ "$client" == "$server" ]]; then
+            echo "Client and server versions match"
+          else
+            echo "Client and server versions are mismatched"
+            exit 1
+          fi
+
+          tag="$(make tag)"
+          if [[ "$client" == "$tag" ]]; then
+            echo "Infra and make tag match"
+          else
+            echo "Infra and make tag are mismatched"
+            exit 1
+          fi
+
+          kill %1

DEPLOYMENT.md

@@ -1,67 +1,5 @@
 # Infra Deployment
 
-## Deploy to an adhoc development cluster
-
-For example one created with `infractl create gke-default`.
-
-To deploy to such a cluster simply:
-
-```
-make deploy-local
-```
-
-The infra server should start and argo should deploy.
-
-```
-$ kubectl -n infra get pods
-NAME                                      READY   STATUS    RESTARTS   AGE
-infra-server-deployment-5c6cfb69c-54k6x   1/1     Running   0          11s
-$ kubectl -n argo get pods
-NAME                                        READY   STATUS    RESTARTS   AGE
-argo-server-58bf6d4f79-cc96j                1/1     Running   1          95s
-workflow-controller-6487cc4688-cdbfz        1/1     Running   0          95s
-```
-
-To connect to the infra-server run a proxy:
-
-```
-kubectl -n infra port-forward svc/infra-server-service 8443:8443
-```
-
-Then use *safari* to connect to the UI if needed. (note: chrome will not accept
-the infra self-signed cert).
-
-Or the locally compiled infractl binary:
-
-```
-bin/infractl-darwin-amd64 -k -e localhost:8443 whoami
-```
-
-### Notes
-
-For clusters created in the `srox-temp-dev-test` to be able to pull images from
-the `stackrox-infra` `us.gcr.io` and `gcr.io` registries, the
-`srox-temp-dev-test` default compute service account requires *Storage Object Viewer* access to
-`artifacts.stackrox-infra.appspot.com` and
-`us.artifacts.stackrox-infra.appspot.com`.
-
-For other clusters e.g. `docker-desktop` image pull secrets will work after the
-deployment has created the namespaces. e.g.
-
-```
-kubectl create secret docker-registry infra-us-gcr-access --docker-server=us.gcr.io --docker-username=_json_key \
-    --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com
-kubectl create secret docker-registry infra-gcr-access --docker-server=gcr.io --docker-username=_json_key \
-    --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com
-kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "infra-gcr-access"},{"name": "infra-us-gcr-access"}]}'
-
-kubectl -n infra create secret docker-registry infra-us-gcr-access --docker-server=us.gcr.io --docker-username=_json_key \
-    --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com
-kubectl -n infra create secret docker-registry infra-gcr-access --docker-server=gcr.io --docker-username=_json_key \
-    --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com
-kubectl -n infra patch serviceaccount default -p '{"imagePullSecrets": [{"name": "infra-gcr-access"},{"name": "infra-us-gcr-access"}]}'
-```
-
 ## Production and Staging Clusters
 
 To work with either of the clusters in `project=stackrox-infra` you will need to either be a member of the `team-automation` group or have someone add you as a project owner.
@@ -154,7 +92,7 @@ correct tooling installed with:
 
 `make image push`
 
-### Development
+### Development/Staging
 
 To render a copy of the charts (for inspection), run:
 

Makefile

@@ -3,11 +3,16 @@ export GO111MODULE=on
 .PHONY: all
 all: image
 
-TAG=$(shell git describe --tags --abbrev=10 --dirty --long)
+TAG=$(shell git describe --tags --abbrev=10 --long)
 .PHONY: tag
 tag:
 	@echo $(TAG)
 
+IMAGE=us.gcr.io/stackrox-infra/infra-server:$(TAG)
+.PHONY: image-name
+image-name:
+	@echo $(IMAGE)
+
 ###########
 ## Build ##
 ###########
@@ -50,7 +55,7 @@ image: server cli ui clean-image
 	@cp bin/infractl-darwin-amd64 image/static/downloads
 	@cp bin/infractl-darwin-arm64 image/static/downloads
 	@cp bin/infractl-linux-amd64 image/static/downloads
-	docker build -t us.gcr.io/stackrox-infra/infra-server:$(TAG) image
+	docker build -t $(IMAGE) image
 
 .PHONY: clean-image
 clean-image:
@@ -167,16 +172,23 @@ configuration-upload:
 	@echo "Uploading configuration to gs://infra-configuration/latest/"
 	gsutil -m cp -R chart/infra-server/configuration "gs://infra-configuration/latest/"
 
+# Combines configuration/{development,production} files into single helm value.yaml files
+# (configuration/{development,production}-values-from-files.yaml) that can be used in template
+# rendering.
+.PHONY: create-consolidated-values
+create-consolidated-values:
+	@./scripts/create-consolidated-values.sh
+
 .PHONY: push
 push:
-	docker push us.gcr.io/stackrox-infra/infra-server:$(TAG) | cat
+	docker push $(IMAGE) | cat
 
 .PHONY: clean-render
 clean-render:
 	@rm -rf chart-rendered
 
 .PHONY: render-local
-render-local: clean-render
+render-local: clean-render create-consolidated-values
 	@if [[ ! -e chart/infra-server/configuration ]]; then \
 		echo chart/infra-server/configuration is absent. Try:; \
 		echo make configuration-download; \
@@ -187,34 +199,37 @@ render-local: clean-render
 	    --output-dir chart-rendered \
 		--set deployment="local" \
 		--set tag="$(TAG)" \
-		--values chart/infra-server/configuration/development-values.yaml
+		--values chart/infra-server/configuration/development-values.yaml \
+		--values chart/infra-server/configuration/development-values-from-files.yaml
 
 .PHONY: render-development
-render-development: clean-render
+render-development: clean-render create-consolidated-values
 	@mkdir -p chart-rendered
 	helm template chart/infra-server \
 	    --output-dir chart-rendered \
 		--set deployment="development" \
 		--set tag="$(TAG)" \
-		--values chart/infra-server/configuration/development-values.yaml
+		--values chart/infra-server/configuration/development-values.yaml \
+		--values chart/infra-server/configuration/development-values-from-files.yaml
 
 .PHONY: render-production
-render-production: clean-render
+render-production: clean-render create-consolidated-values
 	@mkdir -p chart-rendered
 	helm template chart/infra-server \
 	    --output-dir chart-rendered \
 		--set deployment="production" \
 		--set tag="$(TAG)" \
-		--values chart/infra-server/configuration/production-values.yaml
+		--values chart/infra-server/configuration/production-values.yaml \
+		--values chart/infra-server/configuration/production-values-from-files.yaml
 
 dev_context = gke_stackrox-infra_us-west2_infra-development
 prod_context = gke_stackrox-infra_us-west2_infra-production
 this_context = $(shell kubectl config current-context)
 kcdev = kubectl --context $(dev_context)
 kcprod = kubectl --context $(prod_context)
 
-.PHONY: install-local
-install-local:
+.PHONY: install-local-common
+install-local-common:
 	@if [[ "$(this_context)" == "$(dev_context)" ]]; then \
 		echo Your kube context is set to development infra, should be a local cluster; \
 		exit 1; \
@@ -223,18 +238,41 @@ install-local:
 		echo Your kube context is set to production infra, should be a local cluster; \
 		exit 1; \
 	fi
-	@if ! kubectl get ns argo; then \
+	@if ! kubectl get ns argo 2> /dev/null; then \
 		kubectl create namespace argo; \
 		kubectl apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.3.9/install.yaml; \
 	fi
-	@if ! kubectl get ns infra; then \
+	@if ! kubectl get ns infra 2> /dev/null; then \
 		kubectl apply \
-			-f chart-rendered/infra-server/templates/namespace.yaml; \
+			-f chart/infra-server/templates/namespace.yaml; \
 		sleep 10; \
 	fi
+
+.PHONY: install-local
+install-local: install-local-common
 	kubectl apply -R \
 	    -f chart-rendered/infra-server
 
+.PHONY: install-local-without-write
+install-local-without-write: install-local-common
+	gsutil cat gs://infra-configuration/latest/configuration/development-values.yaml \
+               gs://infra-configuration/latest/configuration/development-values-from-files.yaml | \
+	helm template chart/infra-server \
+		--set deployment="local" \
+		--set tag="$(TAG)" \
+		--values - | \
+	kubectl apply -R \
+	    -f -
+	# Bounce the infra-server to ensure proper update
+	@sleep 5
+	kubectl -n infra delete pods -l app=infra-server
+
+.PHONY: local-data-dev-cycle
+local-data-dev-cycle: render-local install-local
+	# Bounce the infra-server to ensure proper update
+	@sleep 5
+	kubectl -n infra delete pods -l app=infra-server
+
 .PHONY: diff-development
 diff-development: render-development
 	$(kcdev) diff -R \
@@ -318,6 +356,7 @@ update-version:
 # i.e. nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 &
 .PHONY: pull-infractl-from-dev-server
 pull-infractl-from-dev-server:
+	@mkdir -p bin
 	@rm -f bin/infractl
 	set -o pipefail; \
 	curl --retry 3 --insecure --silent --show-error --fail --location https://localhost:8443/v1/cli/linux/amd64/upgrade \

README.md

@@ -8,7 +8,16 @@
 
 ## Development
 
-Infra (the server) and infractl (the cli) are written in Go, and use gRPC for client-server communication.
+Infra (the server) and infractl (the cli) are written in Go, and use gRPC for
+client-server communication. The UI uses a React/Typescript/Yarn toolchain (see
+(ui/README.md)[ui/README.md]).
+
+While a development workflow can be achieved using a locally installed
+toolchain, it is also possible to rely on CI. CI will lint, build and push the
+infra server. And then deploy it to a development cluster created using the
+production infra deployment. A
+(comment)[https://github.com/stackrox/infra/pull/711#issuecomment-1270457578]
+will appear on PRs with more detail. 
 
 ### Regenerate Go bindings from protos
 
@@ -32,11 +41,11 @@ correct tooling installed with:
 
 ## Deployment
 
-For additional information on how this service is deployed, please refer to the [deployment instructions](https://github.com/stackrox/infra/blob/master/DEPLOYMENT.md).
+For additional information on how this service is deployed, please refer to the [deployment instructions](DEPLOYMENT.md).
 
 ## Runbook
 
-For additional information on how to debug and remediate issues with the deployed service, please refer to the [runbook instructions](https://github.com/stackrox/infra/blob/master/TROUBLESHOOTING.md).
+For additional information on how to debug and remediate issues with the deployed service, please refer to the [runbook instructions](TROUBLESHOOTING.md).
 
 [circleci-badge]: https://circleci.com/gh/stackrox/infra.svg?style=shield&circle-token=afa342906b658b5349c68b70fa82fd85d1422212
 [circleci-link]:  https://circleci.com/gh/stackrox/infra

chart/infra-server/templates/_helpers.tpl

@@ -1,12 +1,3 @@
-{{- define "require-file" }}
-  {{- $context := (last .) -}}
-  {{- $filename := (first .) -}}
-  {{- $full_filename := (printf "configuration/%s/%s" (required "A valid .Values.environment entry is required!" $context.Values.environment) $filename) -}}
-  {{- if not ($context.Files.Get $full_filename) -}}
-    {{- fail (printf "Failed to locate the file %q." $full_filename) -}}
-  {{- end -}}
-  {{ printf "%s" ($context.Files.Get $full_filename) }}
-{{- end }}
 
 {{- define "docker-io-pull-secret" }}
   {{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.pullSecrets.docker.registry (printf "%s:%s" .Values.pullSecrets.docker.username .Values.pullSecrets.docker.password | b64enc) | b64enc }}

chart/infra-server/templates/argo/secrets.yaml

@@ -38,6 +38,6 @@ metadata:
 
 data:
   credentials.json: |-
-    {{- include "require-file" (list "google-calendar-credentials.json" .) | b64enc | nindent 4 }}
+    {{ required ".Values.google_calendar_credentials_json is undefined" .Values.google_calendar_credentials_json }}
 
 ---

chart/infra-server/templates/demo-certifier.yaml

@@ -10,7 +10,7 @@ metadata:
 
 data:
   google-credentials.json: |-
-    {{- include "require-file" (list "demo/demo-cert-bot.json" .) | b64enc | nindent 4 }}
+    {{ required ".Values.demo__demo_cert_bot_json is undefined" .Values.demo__demo_cert_bot_json }}
 
 ---
 

chart/infra-server/templates/demo/secrets.yaml

@@ -11,11 +11,11 @@ metadata:
 data:
   # Service account used for provisioning demo infrastructure.
   google-credentials.json: |-
-    {{- include "require-file" (list "demo/demo-provisioner.json" .) | b64enc | nindent 4 }}
+    {{ required ".Values.demo__demo_provisioner_json is undefined" .Values.demo__demo_provisioner_json }}
 
   # Service account used for demo GCR integration.
   google-scanner-credentials.json: |-
-    {{- include "require-file" (list "demo/google-scanner-credentials.json" .) | b64enc | nindent 4 }}
+    {{ required ".Values.demo__google_scanner_credentials_json is undefined" .Values.demo__google_scanner_credentials_json }}
 
 ---