🏃 [ci-conformance] Fix kustomize template to better handle envsubst

Makefile

@@ -44,22 +44,22 @@ BIN_DIR := bin
 
 # Binaries.
 CLUSTERCTL := $(BIN_DIR)/clusterctl
-KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize)
+KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen
 ENVSUBST := $(TOOLS_BIN_DIR)/envsubst
 GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
 RELEASE_NOTES_BIN := bin/release-notes
 RELEASE_NOTES := $(TOOLS_DIR)/$(RELEASE_NOTES_BIN)
 
 # Define Docker related variables. Releases should modify and double check these vars.
-GCP_PROJECT ?= $(shell gcloud config get-value project)
+export GCP_PROJECT ?= $(shell gcloud config get-value project)
 REGISTRY ?= gcr.io/$(GCP_PROJECT)
 STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-gcp
 PROD_REGISTRY := us.gcr.io/k8s-artifacts-prod/cluster-api-gcp
 IMAGE_NAME ?= cluster-api-gcp-controller
-CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
-TAG ?= dev
-ARCH ?= amd64
+export CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
+export TAG ?= dev
+export ARCH ?= amd64
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
 
 # Allow overriding manifest generation destination directory
@@ -267,46 +267,65 @@ release-notes: $(RELEASE_NOTES)
 
 # This is used in the get-kubeconfig call below in the create-cluster target. It may be overridden by the
 # e2e-conformance.sh script, which is why we need it as a variable here.
+
 CLUSTER_NAME ?= test1
 
-.PHONY: create-cluster
-create-cluster: $(CLUSTERCTL) $(KUSTOMIZE) $(ENVSUBST) ## Create a development Kubernetes cluster on GCP in a KIND management cluster.
+.PHONY: create-management-cluster
+create-management-cluster: $(KUSTOMIZE) $(ENVSUBST)
+	## Create kind management cluster.
 	kind create cluster --name=clusterapi
-	@if [ ! -z "${LOAD_IMAGE}" ]; then \
-		echo "loading ${LOAD_IMAGE} into kind cluster ..." && \
-		kind --name="clusterapi" load docker-image "${LOAD_IMAGE}"; \
-	fi
+
 	# Install cert manager and wait for availability
 	kubectl create -f https://github.com/jetstack/cert-manager/releases/download/v0.11.1/cert-manager.yaml
 	kubectl wait --for=condition=Available --timeout=5m apiservice v1beta1.webhook.cert-manager.io
 
 	# Deploy CAPI
-	kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.2/cluster-api-components.yaml
+	kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.3/cluster-api-components.yaml
 
 	# Deploy CAPG
+	kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=clusterapi
 	$(KUSTOMIZE) build config | $(ENVSUBST) | kubectl apply -f -
 
 	# Wait for CAPI pods
 	kubectl wait --for=condition=Ready --timeout=5m -n capi-system pod -l cluster.x-k8s.io/provider=cluster-api
 	kubectl wait --for=condition=Ready --timeout=5m -n capi-kubeadm-bootstrap-system pod -l cluster.x-k8s.io/provider=bootstrap-kubeadm
 	kubectl wait --for=condition=Ready --timeout=5m -n capi-kubeadm-control-plane-system pod -l cluster.x-k8s.io/provider=control-plane-kubeadm
 
-	# Wait for CAPG pod
-	kubectl wait --for=condition=Ready --timeout=5m -n capg-system pod -l control-plane=capg-controller-manager
+	# Wait for CAPG pods
+	kubectl wait --for=condition=Ready --timeout=5m -n capg-system pod -l cluster.x-k8s.io/provider=infrastructure-gcp
 
-	# Create Cluster.
+	# required sleep for when creating management and workload cluster simultaneously
 	sleep 10
-	kustomize build templates | $(ENVSUBST) | kubectl apply -f -
+	@echo 'Set kubectl context to the kind management cluster by running "kubectl config set-context kind-clusterapi"'
+
+.PHONY: create-workload-cluster
+create-workload-cluster: $(KUSTOMIZE) $(ENVSUBST)
+	# Create workload Cluster.
+	$(KUSTOMIZE) build templates | $(ENVSUBST) | kubectl apply -f -
 
 	# Wait for the kubeconfig to become available.
-	timeout 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
+	timeout 5m bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
 	# Get kubeconfig and store it locally.
 	kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
-	timeout 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
+	timeout 15m bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
 
 	# Deploy calico
 	kubectl --kubeconfig=./kubeconfig apply -f https://docs.projectcalico.org/manifests/calico.yaml
 
+	@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'
+
+.PHONY: create-cluster
+create-cluster: create-management-cluster create-workload-cluster ## Create a development Kubernetes cluster on GCP in a KIND management cluster.
+
+.PHONY: delete-workload-cluster
+delete-workload-cluster: ## Deletes the example workload Kubernetes cluster
+	@echo 'Your GCP resources will now be deleted, this can take up to 20 minutes'
+	kubectl delete cluster $(CLUSTER_NAME)
+
+.PHONY: delete-cluster
+delete-cluster: delete-workload-cluster  ## Deletes the example kind cluster "clusterapi"
+	kind delete cluster --name=clusterapi
+
 .PHONY: kind-reset
 kind-reset: ## Destroys the "clusterapi" kind cluster.
 	kind delete cluster --name=clusterapi || true

controllers/gcpcluster_controller.go

@@ -50,7 +50,7 @@ func (r *GCPClusterReconciler) SetupWithManager(mgr ctrl.Manager, options contro
 		WithOptions(options).
 		For(&infrav1.GCPCluster{}).
 		Watches(
-			&source.Kind{Type: &infrav1.GCPCluster{}},
+			&source.Kind{Type: &infrav1.GCPMachine{}},
 			&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.GCPMachineToGCPCluster)},
 		).
 		WithEventFilter(pausePredicates).

hack/ci/e2e-conformance.sh

@@ -31,7 +31,6 @@ KUBERNETES_VERSION="v${KUBERNETES_MAJOR_VERSION}.${KUBERNETES_MINOR_VERSION}.${K
 TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%SZ")
 
 ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
 
 # dump logs from kind and all the nodes
 dump-logs() {
@@ -161,6 +160,7 @@ cleanup() {
 
 # our exit handler (trap)
 exit-handler() {
+  unset KUBECONFIG
   dump-logs
   cleanup
 }
@@ -190,7 +190,7 @@ function ssh-to-node() {
 init_image() {
   if [[ "${REUSE_OLD_IMAGES:-false}" == "true" ]]; then
     image=$(gcloud compute images list --project "$GCP_PROJECT" \
-      --no-standard-images --filter="family:capi-ubuntu-1804-k8s-v1-16" --format="table[no-heading](name)")
+      --no-standard-images --filter="family:capi-ubuntu-1804-k8s-v${KUBERNETES_MAJOR_VERSION}-${KUBERNETES_MINOR_VERSION}" --format="table[no-heading](name)")
     if [[ -n "$image" ]]; then
       return
     fi
@@ -224,10 +224,12 @@ init_image() {
   fi
     cat << EOF > "$(go env GOPATH)/src/sigs.k8s.io/image-builder/images/capi/override.json"
 {
+  "build_timestamp": "0",
+  "kubernetes_source_type": "url",
+  "kubernetes_cni_source_type": "url",
+  "kubernetes_http_source": "https://storage.googleapis.com/kubernetes-release-dev/ci",
   "kubernetes_series": "v${KUBERNETES_MAJOR_VERSION}.${KUBERNETES_MINOR_VERSION}",
-  "kubernetes_semver": "${KUBERNETES_VERSION}",
-  "kubernetes_rpm_version": "${KUBERNETES_MAJOR_VERSION}.${KUBERNETES_MINOR_VERSION}.${KUBERNETES_PATCH_VERSION}-0",
-  "kubernetes_deb_version": "${KUBERNETES_MAJOR_VERSION}.${KUBERNETES_MINOR_VERSION}.${KUBERNETES_PATCH_VERSION}-00"
+  "kubernetes_semver": "${KUBERNETES_VERSION}"
 }
 EOF
   if [[ $EUID -ne 0 ]]; then
@@ -245,8 +247,8 @@ EOF
   fi
 }
 
-# build kubernetes / node image, e2e binaries
-build() {
+# build Kubernetes E2E binaries
+build_k8s() {
   # possibly enable bazel build caching before building kubernetes
   if [[ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" == "true" ]]; then
     create_bazel_cache_rcs.sh || true
@@ -270,38 +272,22 @@ build() {
   popd
 }
 
-# generate manifests needed for creating the GCP cluster to run the tests
-generate_manifests() {
-  if ! command -v kustomize >/dev/null 2>&1; then
-    (cd ./hack/tools/ && GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3)
-  fi
-
-  (GCP_PROJECT=${GCP_PROJECT} \
-  PULL_POLICY=Never \
-    make modules docker-build)
-
-  # Enable the bits to inject a script that can pull newer versions of kubernetes
-  if [[ -n ${CI_VERSION:-} || -n ${USE_CI_ARTIFACTS:-} ]]; then
-    if ! grep -i -wq "patchesStrategicMerge" "templates/kustomization.yaml"; then
-      echo "patchesStrategicMerge:" >> "templates/kustomization.yaml"
-      echo "- kustomizeversions.yaml" >> "templates/kustomization.yaml"
-    fi
-  fi
-}
-
 # up a cluster with kind
 create_cluster() {
   # actually create the cluster
   KIND_IS_UP=true
 
+  filter="name~cluster-api-ubuntu-1804-${KUBERNETES_VERSION//[.+]/-}"
+  image_id=$(gcloud compute images list --project "$GCP_PROJECT" \
+    --no-standard-images --filter="${filter}" --format="table[no-heading](name)")
+  if [[ -z "$image_id" ]]; then
+    echo "unable to find image using : $filter $GCP_PROJECT ... bailing out!"
+    exit 1
+  fi
+
   tracestate="$(shopt -po xtrace)"
   set +o xtrace
 
-  if [[ -n ${USE_CI_ARTIFACTS:-} ]]; then
-    # TODO: revert to https://dl.k8s.io/ci/latest-green.txt once https://github.com/kubernetes/release/issues/897 is fixed.
-    CI_VERSION=${CI_VERSION:-$(curl -sSL https://dl.k8s.io/ci/k8s-master.txt)}
-  fi
-
   # Load the newly built image into kind and start the cluster
   (GCP_REGION=${GCP_REGION} \
   GCP_PROJECT=${GCP_PROJECT} \
@@ -313,8 +299,8 @@ create_cluster() {
   GCP_NETWORK_NAME=${GCP_NETWORK_NAME} \
   GCP_B64ENCODED_CREDENTIALS=$(base64 -w0 "$GOOGLE_APPLICATION_CREDENTIALS") \
   CLUSTER_NAME="${CLUSTER_NAME}" \
-  CI_VERSION="${CI_VERSION:-}" \
-  LOAD_IMAGE="gcr.io/${GCP_PROJECT}/cluster-api-gcp-controller-amd64:dev" \
+  CI_VERSION=${CI_VERSION} \
+  IMAGE_ID="projects/${GCP_PROJECT}/global/images/${image_id}" \
     make create-cluster)
 
   eval "$tracestate"
@@ -414,6 +400,17 @@ init_networks() {
     --nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips
 }
 
+# generate manifests needed for creating the GCP cluster to run the tests
+add_kustomize_patch() {
+    # Enable the bits to inject a script that can pull newer versions of kubernetes
+    if ! grep -i -wq "patchesStrategicMerge" "templates/kustomization.yaml"; then
+        echo "patchesStrategicMerge:" >> "templates/kustomization.yaml"
+    fi
+    if ! grep -i -wq "kustomizeversions" "templates/kustomization.yaml"; then
+        echo "- kustomizeversions.yaml" >> "templates/kustomization.yaml"
+    fi
+}
+
 # setup kind, build kubernetes, create a cluster, run the e2es
 main() {
   for arg in "$@"
@@ -435,15 +432,15 @@ main() {
 
   if [[ -z "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then
     cat <<EOF
-$GOOGLE_APPLICATION_CREDENTIALS is not set.
+GOOGLE_APPLICATION_CREDENTIALS is not set.
 Please set this to the path of the service account used to run this script.
 EOF
     return 2
   else
     gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
   fi
   if [[ -z "$GCP_PROJECT" ]]; then
-    GCP_PROJECT=$(cat "${GOOGLE_APPLICATION_CREDENTIALS}" | jq -r .project_id)
+    GCP_PROJECT=$(jq -r .project_id "${GOOGLE_APPLICATION_CREDENTIALS}")
     cat <<EOF
 GCP_PROJECT is not set. Using project_id $GCP_PROJECT
 EOF
@@ -466,22 +463,47 @@ EOF
   export ARTIFACTS
   mkdir -p "${ARTIFACTS}/logs"
 
-  source "${REPO_ROOT}/hack/ensure-go.sh"
-  source "${REPO_ROOT}/hack/ensure-kind.sh"
+  # Initialize the necessary network requirements
+  if [[ -n "${SKIP_INIT_NETWORK:-}" ]]; then
+    echo "Skipping network initialization..."
+  else
+    init_networks
+  fi
+
+  if [[ -n ${CI_VERSION:-} || -n ${USE_CI_ARTIFACTS:-} ]]; then
+    CI_VERSION=${CI_VERSION:-$(curl -sSL https://dl.k8s.io/ci/k8s-master.txt)}
+    KUBERNETES_VERSION=${CI_VERSION}
+    KUBERNETES_MAJOR_VERSION=$(echo "${KUBERNETES_VERSION}" | cut -d '.' -f1 - | sed 's/v//')
+    KUBERNETES_MINOR_VERSION=$(echo "${KUBERNETES_VERSION}" | cut -d '.' -f2 -)
+  fi
 
-  # now build and run the cluster and tests
-  init_networks
-  build
-  generate_manifests
   if [[ -n "${SKIP_INIT_IMAGE:-}" ]]; then
-    echo "Skipping image initialization..."
+    echo "Skipping GCP image initialization..."
   else
     init_image
   fi
 
-  create_cluster
+  # Build the images
+  if [[ -n "${SKIP_IMAGE_BUILD:-}" ]]; then
+    echo "Skipping Container image building..."
+  else
+    (GCP_PROJECT=${GCP_PROJECT} PULL_POLICY=Never make modules docker-build)
+  fi
+
+  # create cluster
+  if [[ -n "${SKIP_CREATE_CLUSTER:-}" ]]; then
+    echo "Skipping cluster creation..."
+  else
+    if [[ -n ${CI_VERSION:-} ]]; then
+      echo "Adding kustomize patch for ci version..."
+      add_kustomize_patch
+    fi
+    create_cluster
+  fi
 
-  if [[ -z "${SKIP_RUN_TESTS:-}" ]]; then
+  # build k8s binaries and run conformance tests
+  if [[ -z "${SKIP_TESTS:-}" && -z "${SKIP_RUN_TESTS:-}" ]]; then
+    build_k8s
     run_tests
   fi
 }

hack/ensure-kustomize.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+GOPATH_BIN="$(go env GOPATH)/bin/"
+MINIMUM_KUSTOMIZE_VERSION=3.1.0
+
+# Ensure the kustomize tool exists and is a viable version, or installs it
+verify_kustomize_version() {
+
+  # If kustomize is not available on the path, get it
+  if ! [ -x "$(command -v kustomize)" ]; then
+    if [[ "${OSTYPE}" == "linux-gnu" ]]; then
+      echo 'kustomize not found, installing'
+      if ! [ -d "${GOPATH_BIN}" ]; then
+        mkdir -p "${GOPATH_BIN}"
+      fi
+      curl -sLo "${GOPATH_BIN}/kustomize" https://github.com/kubernetes-sigs/kustomize/releases/download/v${MINIMUM_KUSTOMIZE_VERSION}/kustomize_${MINIMUM_KUSTOMIZE_VERSION}_linux_amd64
+      chmod +x "${GOPATH_BIN}/kustomize"
+    else
+      echo "Missing required binary in path: kustomize"
+      return 2
+    fi
+  fi
+
+  local kustomize_version
+  kustomize_version=$(kustomize version)
+  if [[ "${MINIMUM_KUSTOMIZE_VERSION}" != $(echo -e "${MINIMUM_KUSTOMIZE_VERSION}\n${kustomize_version}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) ]]; then
+    cat <<EOF
+Detected kustomize version: ${kustomize_version}.
+Requires ${MINIMUM_KUSTOMIZE_VERSION} or greater.
+Please install ${MINIMUM_KUSTOMIZE_VERSION} or later.
+EOF
+    return 2
+  fi
+}
+
+verify_kustomize_version