Switch skaffold to use Kustomize

Summary: This is a sequence of diffs converting our templated YAMLs to Kustomize. This converts skaffold to use Kustomize. More changes to delete old yamls and switch dev environment are coming. Test Plan: Jenkins Reviewers: oazizi, michelle, philkuz, #engineering Reviewed By: oazizi, #engineering Differential Revision: https://phab.corp.pixielabs.ai/D1143 GitOrigin-RevId: 587f21c
svc-design · Jul 10, 2019 · dcbd6a7 · dcbd6a7
1 parent 9288ef4
commit dcbd6a7
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 46 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -474,7 +474,8 @@ def buildScriptForNightly = {
         dockerStepWithBazelDeps('', devDockerImageExtrasWithTag) {
           withKubeConfig([credentialsId: K8S_CREDS_NAME,
                           serverUrl: K8S_ADDR, namespace: K8S_NS]) {
-            sh 'PL_IMAGE_TAG=nightly-$(date +%s)-`cat SOURCE_VERSION` make skaffold-staging'
+            sh 'PL_IMAGE_TAG=nightly-$(date +%s)-`cat SOURCE_VERSION` make deploy-vizier-nightly'
+            sh 'PL_IMAGE_TAG=nightly-$(date +%s)-`cat SOURCE_VERSION` make deploy-customer-docs-nightly'
           }
         }
       }

diff --git a/Makefile b/Makefile
@@ -18,8 +18,7 @@ KUBECTL_FLAGS := -n pl
 ## Skaffold command to use.
 SKAFFOLD := skaffold
 
-## The directory to write template files to for skaffold (with respect to bazel info workspace).
-SKAFFOLD_DIR := $$(bazel info workspace)/skaffold_build
+SKAFFOLD_DIR := $$(bazel info workspace)/skaffold
 
 ## Active operating system (Linux vs MacOS).
 UNAME_S := $(shell uname -s)
@@ -33,7 +32,6 @@ endif
 .PHONY: clean
 clean:
 	$(BAZEL) clean
-	rm -rf $(SKAFFOLD_DIR)
 
 .PHONY: pristine
 pristine:
@@ -126,21 +124,11 @@ dev-env-stop: ## Stop dev environment.
 dev-env-teardown: dev-env-stop ## Clean up dev environment.
 	$(MINIKUBE) delete
 
-skaffold-dev: ## Run Skaffold in the dev environment.
-	$(BAZEL) run //templates/skaffold:skaffoldtemplate -- --build_dir $(SKAFFOLD_DIR) --build_type dev
-	$(SKAFFOLD) dev -f $(SKAFFOLD_DIR)/skaffold/skaffold_dev.yaml
+deploy-vizier-nightly: ## Deploy vizier in nightly environment.
+	PL_BUILD_TYPE=nightly $(SKAFFOLD) run -f $(SKAFFOLD_DIR)/skaffold_nightly.yaml
 
-skaffold-prod: ## Run Skaffold in the prod environment.
-	$(BAZEL) run //templates/skaffold:skaffoldtemplate -- --build_dir $(SKAFFOLD_DIR) --build_type prod
-	$(SKAFFOLD) run -f $(SKAFFOLD_DIR)/skaffold/skaffold_prod.yaml
-
-skaffold-staging: ## Run Skaffold in the staging environment.
-	$(BAZEL) run //templates/skaffold:skaffoldtemplate -- --build_dir $(SKAFFOLD_DIR) --build_type staging
-	$(SKAFFOLD) run -f $(SKAFFOLD_DIR)/skaffold/skaffold_staging.yaml
-
-skaffold-nightly: ## Run Skaffold in the nightly environment.
-	$(BAZEL) run //templates/skaffold:skaffoldtemplate -- --build_dir $(SKAFFOLD_DIR) --build_type nightly
-	PL_BUILD_TYPE=nightly $(SKAFFOLD) run -f $(SKAFFOLD_DIR)/skaffold/skaffold_nightly.yaml
+deploy-customer-docs-nightly: ## Deploy customer docs in nightly environment.
+	PL_BUILD_TYPE=nightly $(SKAFFOLD) run -f $(SKAFFOLD_DIR)/skaffold_customer_docs.yaml
 
 gen-jwt: ## Generate a JWT for our demo cluster.
 	@JWT=$$(PL_JWT_SIGNING_KEY=ABCDEFG $(BAZEL) run //src/utils/gen_test_key); \

diff --git a/skaffold/skaffold_customer_docs.yaml b/skaffold/skaffold_customer_docs.yaml
@@ -0,0 +1,42 @@
+apiVersion: skaffold/v1beta8
+kind: Config
+build:
+  artifacts:
+  - image: gcr.io/pl-dev-infra/customer_docs_prod_image
+    context: .
+    bazel:
+      target: //docs/customer:customer_docs_prod_image.tar
+deploy:
+  kustomize:
+    path: k8s/customer_docs_dev
+profiles:
+  - name: dev
+    activation:
+      - command: dev
+    patches:
+      # Don't push images in the dev build.
+      - op: replace
+        path: /build
+        value:
+          local:
+            push: false
+
+  - name: nightly
+    activation:
+      - env: PL_BUILD_TYPE=nightly
+    patches:
+      # Set compilation to be opt build.
+      - op: add
+        path: /build/artifacts/context=./bazel/args
+        value: ["--compilation_mode=opt"]
+
+      - op: set
+        path: /build/tagPolicy/envTemplate
+        value:
+          template: "{{.IMAGE_NAME}}:{{.PL_IMAGE_TAG}}"
+
+      # Set path to be nightly yamls.
+      - op: replace
+        path: /deploy/kustomize
+        value:
+          path: k8s/customer_docs/nightly
diff --git a/skaffold/skaffold.skfld.tmpl → skaffold/skaffold_vizier.yaml b/skaffold/skaffold.skfld.tmpl → skaffold/skaffold_vizier.yaml
@@ -1,11 +1,6 @@
-# Generated file: bazel run //templates/skaffold:skaffoldtemplate
-# DO NOT EDIT
 apiVersion: skaffold/v1beta8
 kind: Config
 build:
-  tagPolicy:
-    envTemplate:
-      template: "{{`{{.IMAGE_NAME}}:{{.PL_IMAGE_TAG}}`}}"
   artifacts:
   - image: gcr.io/pl-dev-infra/agent_image
     context: .
@@ -34,22 +29,14 @@ build:
   local:
     push: true
 deploy:
-  kubectl:
-    manifests:
-    - {{.BuildDir}}/src/vizier/services/agent/agent_{{.Deployment}}.yaml
-    - {{.BuildDir}}/src/vizier/services/query_broker/service_{{.Deployment}}.yaml
-    - {{.BuildDir}}/src/vizier/services/metadata/service_{{.Deployment}}.yaml
-    - {{.BuildDir}}/src/services/api/service_{{.Deployment}}.yaml
-    - {{.BuildDir}}/src/services/api/service_{{.Deployment}}.yaml
-    - {{.BuildDir}}/src/services/proxy/service_{{.Deployment}}.yaml
+  kustomize:
+    path: k8s/vizier/dev
 profiles:
   - name: dev
     activation:
       - command: dev
     patches:
       # Use default tag policy (hash) for dev builds.
-      - op: remove
-        path: /build/tagPolicy/envTemplate
       # Don't push images in the dev build.
       - op: replace
         path: /build
@@ -61,20 +48,18 @@ profiles:
     activation:
       - env: PL_BUILD_TYPE=nightly
     patches:
-      # Add customer docs to the nightly build.
-      - op: add
-        path: /build/artifacts/-
-        value:
-          image: gcr.io/pl-dev-infra/customer_docs_prod_image
-          context: .
-          bazel:
-            target: //docs/customer:customer_docs_prod_image.tar
-
-      - op: add
-        path: /deploy/kubectl/manifests/-
-        value: {{.BuildDir}}/docs/customer/service_{{.Deployment}}.yaml
-
       # Set compilation to be opt build.
       - op: add
         path: /build/artifacts/context=./bazel/args
         value: ["--compilation_mode=opt"]
+
+      - op: set
+        path: /build/tagPolicy/envTemplate
+        value:
+          template: "{{.IMAGE_NAME}}:{{.PL_IMAGE_TAG}}"
+
+# Set path to be nightly yamls.
+      - op: replace
+        path: /deploy/kustomize
+        value:
+          path: k8s/vizier/nightly