Refactor e2e yaml value files (istio#12076)

* Refactor e2e yaml value files This change involes: * renaming uses of old make target * adding all generated files to gitignore * create new target to build all e2e yaml files and another for the demo files that are included in release * move all testing value files, and example value files, to folders * create value files for tests that were using --set * Fix reference to values-e2e.yaml * Fix typo * Add readme and fix test failures * Fix integration tests file * Enable core dump for auth sds test * Actually use coredump * Move istio minimal - needed for docs * resolve conflict
qfel · Mar 1, 2019 · 872351a · 872351a
1 parent 3a0daf1
commit 872351a
Show file tree

Hide file tree

Showing 26 changed files with 127 additions and 189 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -162,7 +162,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -201,7 +201,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -236,7 +236,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_e2e_test_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -270,7 +270,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_e2e_test_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -309,7 +309,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -369,7 +369,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -406,7 +406,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -440,7 +440,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -497,7 +497,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:

diff --git a/.gitignore b/.gitignore
@@ -57,6 +57,11 @@ install/kubernetes/istio-multicluster.yaml
 install/kubernetes/istio-remote.yaml
 install/kubernetes/istio-mcp.yaml
 install/kubernetes/istio-auth-mcp.yaml
+install/kubernetes/istio-auth-non-mcp.yaml
+install/kubernetes/istio-auth-sds.yaml
+install/kubernetes/istio-init.yaml
+install/kubernetes/istio-non-mcp.yaml
+install/kubernetes/istio-minimal.yaml
 install/kubernetes/helm/istio/requirements.lock
 samples/bookinfo/platform/consul/bookinfo.sidecars.yaml
 *.orig

diff --git a/Makefile b/Makefile
@@ -630,16 +630,6 @@ $(HELM):
 $(HOME)/.helm:
 	$(HELM) init --client-only
 
-# create istio-remote.yaml
-istio-remote.yaml: $(HELM) $(HOME)/.helm
-	cat install/kubernetes/namespace.yaml > install/kubernetes/$@
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/$@
-	$(HELM) template --name=istio --namespace=istio-system \
-		--values install/kubernetes/helm/istio/values-istio-remote.yaml \
-		--set istio_cni.enabled=${ENABLE_ISTIO_CNI} \
-		${EXTRA_HELM_SETTINGS} \
-		install/kubernetes/helm/istio >> install/kubernetes/$@
-
 # create istio-init.yaml
 istio-init.yaml: $(HELM) $(HOME)/.helm
 	cat install/kubernetes/namespace.yaml > install/kubernetes/$@
@@ -649,9 +639,9 @@ istio-init.yaml: $(HELM) $(HOME)/.helm
 		--set global.hub=${HUB} \
 		install/kubernetes/helm/istio-init >> install/kubernetes/$@
 
-# creates istio.yaml istio-auth.yaml istio-one-namespace.yaml istio-one-namespace-auth.yaml istio-one-namespace-trust-domain.yaml
+# creates istio-demo.yaml istio-demo-auth.yaml istio-remote.yaml
 # Ensure that values-$filename is present in install/kubernetes/helm/istio
-isti%.yaml: $(HELM) $(HOME)/.helm
+istio-demo.yaml istio-demo-auth.yaml istio-remote.yaml istio-minimal.yaml: $(HELM) $(HOME)/.helm
 	cat install/kubernetes/namespace.yaml > install/kubernetes/$@
 	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/$@
 	$(HELM) template \
@@ -668,50 +658,28 @@ isti%.yaml: $(HELM) $(HOME)/.helm
 		--values install/kubernetes/helm/istio/values-$@ \
 		install/kubernetes/helm/istio >> install/kubernetes/$@
 
-generate_yaml: $(HELM) $(HOME)/.helm istio-init.yaml
-	./install/updateVersion.sh -a ${HUB},${TAG} >/dev/null 2>&1
-	cat install/kubernetes/namespace.yaml > install/kubernetes/istio.yaml
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/istio.yaml
-	$(HELM) template \
-		--name=istio \
-		--namespace=istio-system \
-		--set global.hub=${HUB} \
-		--set global.tag=${TAG} \
-		--set global.imagePullPolicy=$(PULL_POLICY) \
-		--set global.mtls.enabled=false \
-		--set global.controlPlaneSecurityEnabled=false \
-		--set global.proxy.enableCoreDump=${ENABLE_COREDUMP} \
-		--set istio_cni.enabled=${ENABLE_ISTIO_CNI} \
-		--set gateways.istio-egressgateway.enabled=true \
-		--set global.outboundTrafficPolicy.mode=REGISTRY_ONLY \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		${EXTRA_HELM_SETTINGS} \
-		install/kubernetes/helm/istio >> install/kubernetes/istio.yaml
-
-	cat install/kubernetes/namespace.yaml > install/kubernetes/istio-auth.yaml
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/istio-auth.yaml
-	$(HELM) template \
-		--name=istio \
-		--namespace=istio-system \
-		--set global.hub=${HUB} \
-    --set global.tag=${TAG} \
-		--set global.imagePullPolicy=$(PULL_POLICY) \
-		--set global.mtls.enabled=true \
-		--set global.controlPlaneSecurityEnabled=true \
-		--set global.proxy.enableCoreDump=${ENABLE_COREDUMP} \
-		--set istio_cni.enabled=${ENABLE_ISTIO_CNI} \
-		--set gateways.istio-egressgateway.enabled=true \
-		--set global.outboundTrafficPolicy.mode=REGISTRY_ONLY \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		${EXTRA_HELM_SETTINGS} \
-		install/kubernetes/helm/istio >> install/kubernetes/istio-auth.yaml
-
-generate_yaml_coredump: export ENABLE_COREDUMP=true
-generate_yaml_coredump:
-	$(MAKE) generate_yaml
-
-# TODO(howardjohn) clean all of this up
-istio-auth-mcp.yaml:
+e2e_files = istio-auth-non-mcp.yaml \
+			istio-auth-sds.yaml \
+			istio-non-mcp.yaml \
+			istio.yaml \
+			istio-auth.yaml \
+			istio-auth-mcp.yaml \
+			istio-auth-multicluster.yaml \
+			istio-mcp.yaml \
+			istio-one-namespace.yaml \
+			istio-one-namespace-auth.yaml \
+			istio-one-namespace-trust-domain.yaml \
+			istio-multicluster.yaml \
+
+.PHONY: generate_e2e_yaml generate_e2e_yaml_coredump
+generate_e2e_yaml: $(e2e_files)
+
+generate_e2e_yaml_coredump: export ENABLE_COREDUMP=true
+generate_e2e_yaml_coredump:
+	$(MAKE) generate_e2e_yaml
+
+# Create yaml files for e2e tests. Applies values-e2e.yaml, then values-$filename.yaml
+$(e2e_files): $(HELM) $(HOME)/.helm istio-init.yaml
 	cat install/kubernetes/namespace.yaml > install/kubernetes/$@
 	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/$@
 	$(HELM) template \
@@ -723,84 +691,9 @@ istio-auth-mcp.yaml:
 		--set global.proxy.enableCoreDump=${ENABLE_COREDUMP} \
 		--set istio_cni.enabled=${ENABLE_ISTIO_CNI} \
 		${EXTRA_HELM_SETTINGS} \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		--values install/kubernetes/helm/istio/values-istio-auth-mcp.yaml \
-		install/kubernetes/helm/istio >> install/kubernetes/istio-auth-mcp.yaml
-
-# TODO(sdake) All this copy and paste needs to go.  This is easy to wrap up in
-#             isti%.yaml macro with value files per test scenario.  Will handle
-#             as a followup PR.
-generate_e2e_test_yaml: $(HELM) $(HOME)/.helm istio-init.yaml
-	#./install/updateVersion.sh -a ${HUB},${TAG} >/dev/null 2>&1
-	cat install/kubernetes/namespace.yaml > install/kubernetes/istio.yaml
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/istio.yaml
-	$(HELM) template --set global.tag=${TAG} \
-		--name=istio \
-		--namespace=istio-system \
-		--set global.hub=${HUB} \
-		--set global.proxy.enableCoreDump=${ENABLE_COREDUMP} \
-		--set gateways.istio-egressgateway.enabled=true \
-		--set global.outboundTrafficPolicy.mode=REGISTRY_ONLY \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		${EXTRA_HELM_SETTINGS} \
-		install/kubernetes/helm/istio >> install/kubernetes/istio.yaml
-
-	cat install/kubernetes/namespace.yaml > install/kubernetes/istio-auth.yaml
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/istio-auth.yaml
-	$(HELM) template --set global.tag=${TAG} \
-		--name=istio \
-		--namespace=istio-system \
-		--set global.hub=${HUB} \
-		--set global.mtls.enabled=true \
-		--set global.controlPlaneSecurityEnabled=true \
-		--set global.proxy.enableCoreDump=${ENABLE_COREDUMP} \
-		--set gateways.istio-egressgateway.enabled=true \
-		--set global.outboundTrafficPolicy.mode=REGISTRY_ONLY \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		${EXTRA_HELM_SETTINGS} \
-		install/kubernetes/helm/istio >> install/kubernetes/istio-auth.yaml
-
-	cat install/kubernetes/namespace.yaml > install/kubernetes/istio-non-mcp.yaml
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/istio-non-mcp.yaml
-	$(HELM) template --set global.tag=${TAG} \
-		--name=istio \
-		--namespace=istio-system \
-		--set global.hub=${HUB} \
-		--set global.proxy.enableCoreDump=${ENABLE_COREDUMP} \
-		--set global.useMCP=false \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		${EXTRA_HELM_SETTINGS} \
-		install/kubernetes/helm/istio >> install/kubernetes/istio-non-mcp.yaml
-
-	cat install/kubernetes/namespace.yaml > install/kubernetes/istio-auth-non-mcp.yaml
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/istio-auth-non-mcp.yaml
-	$(HELM) template --set global.tag=${TAG} \
-		--name=istio \
-		--namespace=istio-system \
-		--set global.hub=${HUB} \
-		--set global.mtls.enabled=true \
-		--set global.controlPlaneSecurityEnabled=true \
-		--set global.proxy.enableCoreDump=${ENABLE_COREDUMP} \
-		--set global.useMCP=false \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		${EXTRA_HELM_SETTINGS} \
-		install/kubernetes/helm/istio >> install/kubernetes/istio-auth-non-mcp.yaml
-
-	cat install/kubernetes/namespace.yaml > install/kubernetes/istio-auth-sds.yaml
-	cat install/kubernetes/helm/istio-init/files/crd-* >> install/kubernetes/istio-auth-sds.yaml
-	$(HELM) template --set global.tag=${TAG} \
-		--name=istio \
-		--namespace=istio-system \
-		--set global.hub=${HUB} \
-		--set global.mtls.enabled=true \
-		--set global.proxy.enableCoreDump=true \
-		--set istio_cni.enabled=${ENABLE_ISTIO_CNI} \
-		--set gateways.istio-egressgateway.enabled=true \
-		--set global.outboundTrafficPolicy.mode=REGISTRY_ONLY \
-		${EXTRA_HELM_SETTINGS} \
-		--values install/kubernetes/helm/istio/values-e2e.yaml \
-		--values install/kubernetes/helm/istio/values-istio-sds-auth.yaml \
-		install/kubernetes/helm/istio >> install/kubernetes/istio-auth-sds.yaml
+		--values install/kubernetes/helm/istio/test-values/values-e2e.yaml \
+		--values install/kubernetes/helm/istio/test-values/values-$@ \
+		install/kubernetes/helm/istio >> install/kubernetes/$@
 
 # files generated by the default invocation of updateVersion.sh
 FILES_TO_CLEAN+=install/consul/istio.yaml \

diff --git a/install/kubernetes/helm/istio/example-values/README.md b/install/kubernetes/helm/istio/example-values/README.md
@@ -0,0 +1,5 @@
+# Example Values
+
+These files provide various example values for different Istio setups.
+
+To use them, [read the docs](https://istio.io/docs/setup/kubernetes/helm-install/) and add the flag `--values example-file.yaml`.
diff --git a/...istio/values-istio-example-sds-vault.yaml → ...alues/values-istio-example-sds-vault.yaml b/...istio/values-istio-example-sds-vault.yaml → ...alues/values-istio-example-sds-vault.yaml
diff --git a/...tes/helm/istio/values-istio-gateways.yaml → ...example-values/values-istio-gateways.yaml b/...tes/helm/istio/values-istio-gateways.yaml → ...example-values/values-istio-gateways.yaml
diff --git a/...tes/helm/istio/values-istio-googleca.yaml → ...example-values/values-istio-googleca.yaml b/...tes/helm/istio/values-istio-googleca.yaml → ...example-values/values-istio-googleca.yaml
diff --git a/...o/values-istio-multicluster-gateways.yaml → ...s/values-istio-multicluster-gateways.yaml b/...o/values-istio-multicluster-gateways.yaml → ...s/values-istio-multicluster-gateways.yaml
diff --git a/install/kubernetes/helm/istio/test-values/README.md b/install/kubernetes/helm/istio/test-values/README.md
@@ -0,0 +1,7 @@
+# Test Values
+
+These files are intended to be used to install Istio for E2E tests.
+
+The rendered files can be generated with `make generate_e2e_yaml`.
+
+These files will all have `values-e2e.yaml` applied to them *first*, so if there are settings there that should not be included in the test the must be overridden.
diff --git a/...all/kubernetes/helm/istio/values-e2e.yaml → ...es/helm/istio/test-values/values-e2e.yaml b/...all/kubernetes/helm/istio/values-e2e.yaml → ...es/helm/istio/test-values/values-e2e.yaml
@@ -12,13 +12,17 @@ global:
     enableCoreDump: true
 
   disablePolicyChecks: false
+  outboundTrafficPolicy:
+    mode: REGISTRY_ONLY
 
 prometheus:
   scrapeInterval: 5s
 
 gateways:
   istio-ingressgateway:
     autoscaleMax: 1
+  istio-egressgateway:
+    enabled: true
 
 mixer:
   policy:

diff --git a/...tes/helm/istio/values-istio-auth-mcp.yaml → ...io/test-values/values-istio-auth-mcp.yaml b/...tes/helm/istio/values-istio-auth-mcp.yaml → ...io/test-values/values-istio-auth-mcp.yaml
diff --git a/...istio/values-istio-auth-multicluster.yaml → ...alues/values-istio-auth-multicluster.yaml b/...istio/values-istio-auth-multicluster.yaml → ...alues/values-istio-auth-multicluster.yaml
diff --git a/install/kubernetes/helm/istio/test-values/values-istio-auth-non-mcp.yaml b/install/kubernetes/helm/istio/test-values/values-istio-auth-non-mcp.yaml
@@ -0,0 +1,7 @@
+global:
+  mtls:
+    enabled: true
+
+  controlPlaneSecurityEnabled: true
+
+  useMCP: false
diff --git a/install/kubernetes/helm/istio/test-values/values-istio-auth-sds.yaml b/install/kubernetes/helm/istio/test-values/values-istio-auth-sds.yaml
@@ -0,0 +1,23 @@
+global:
+  controlPlaneSecurityEnabled: false
+
+  mtls:
+    # Default setting for service-to-service mtls. Can be set explicitly using
+    # destination rules or service annotations.
+    enabled: true
+
+  sds:
+    enabled: true
+    udsPath: "unix:/var/run/sds/uds_path"
+    useNormalJwt: true
+
+  proxy:
+    enableCoreDump: true
+
+nodeagent:
+  enabled: true
+  image: node-agent-k8s
+  env:
+    CA_PROVIDER: "Citadel"
+    CA_ADDR: "istio-citadel:8060"
+    VALID_TOKEN: true
diff --git a/...ernetes/helm/istio/values-istio-auth.yaml → .../istio/test-values/values-istio-auth.yaml b/...ernetes/helm/istio/values-istio-auth.yaml → .../istio/test-values/values-istio-auth.yaml
diff --git a/...bernetes/helm/istio/values-istio-mcp.yaml → ...m/istio/test-values/values-istio-mcp.yaml b/...bernetes/helm/istio/values-istio-mcp.yaml → ...m/istio/test-values/values-istio-mcp.yaml
diff --git a/...helm/istio/values-istio-multicluster.yaml → ...est-values/values-istio-multicluster.yaml b/...helm/istio/values-istio-multicluster.yaml → ...est-values/values-istio-multicluster.yaml
diff --git a/install/kubernetes/helm/istio/test-values/values-istio-non-mcp.yaml b/install/kubernetes/helm/istio/test-values/values-istio-non-mcp.yaml
@@ -0,0 +1,2 @@
+global:
+  useMCP: false
diff --git a/...stio/values-istio-one-namespace-auth.yaml → ...lues/values-istio-one-namespace-auth.yaml b/...stio/values-istio-one-namespace-auth.yaml → ...lues/values-istio-one-namespace-auth.yaml
diff --git a/...ues-istio-one-namespace-trust-domain.yaml → ...ues-istio-one-namespace-trust-domain.yaml b/...ues-istio-one-namespace-trust-domain.yaml → ...ues-istio-one-namespace-trust-domain.yaml
diff --git a/...elm/istio/values-istio-one-namespace.yaml → ...st-values/values-istio-one-namespace.yaml b/...elm/istio/values-istio-one-namespace.yaml → ...st-values/values-istio-one-namespace.yaml
diff --git a/...l/kubernetes/helm/istio/values-istio.yaml → .../helm/istio/test-values/values-istio.yaml b/...l/kubernetes/helm/istio/values-istio.yaml → .../helm/istio/test-values/values-istio.yaml
diff --git a/install/updateVersion.sh b/install/updateVersion.sh
@@ -148,15 +148,9 @@ function gen_file() {
 }
 
 function gen_istio_files() {
-    if [[ -n ${ISTIO_RELEASE:-} ]]; then
-        for target in istio-demo.yaml istio-demo-auth.yaml; do
-            gen_file $target "${DEST_DIR}"
-        done
-    else
-        for target in istio.yaml istio-auth.yaml istio-one-namespace.yaml istio-one-namespace-auth.yaml istio-one-namespace-trust-domain.yaml istio-multicluster.yaml istio-auth-multicluster.yaml istio-remote.yaml istio-mcp.yaml istio-auth-mcp.yaml;do
-            gen_file $target "${DEST_DIR}"
-        done
-    fi
+    for target in istio-demo.yaml istio-demo-auth.yaml; do
+        gen_file $target "${DEST_DIR}"
+    done
 }
 
 function update_istio_install_docker() {

diff --git a/pkg/test/framework/runtime/components/environment/kube/settings.go b/pkg/test/framework/runtime/components/environment/kube/settings.go
@@ -42,7 +42,7 @@ const (
 	DefaultSystemNamespace = "istio-system"
 
 	// DefaultValuesFile for Istio Helm deployment.
-	DefaultValuesFile = "values-istio-mcp.yaml"
+	DefaultValuesFile = "test-values/values-istio-mcp.yaml"
 
 	// LatestTag value
 	LatestTag = "latest"