Fix a few items that were breaking container-based build. (#17447)

Makefile

@@ -57,17 +57,18 @@ TARGET_OUT ?= $(HOME)/istio_out/$(REPO_NAME)
 ifeq ($(BUILD_WITH_CONTAINER),1)
 CONTAINER_CLI ?= docker
 DOCKER_SOCKET_MOUNT ?= -v /var/run/docker.sock:/var/run/docker.sock
-IMG ?= gcr.io/istio-testing/build-tools:2019-09-23T12-48-14
+IMG ?= gcr.io/istio-testing/build-tools:2019-09-25T19-39-04
 UID = $(shell id -u)
 PWD = $(shell pwd)
 
+$(info Building with the build container: $(IMG).)
+
 # Determine the timezone across various platforms to pass into the
 # docker run operation. This operation assumes zoneinfo is within
 # the path of the file.
 TIMEZONE=`readlink $(READLINK_FLAGS) /etc/localtime | sed -e 's/^.*zoneinfo\///'`
 
 RUN = $(CONTAINER_CLI) run -t -i --sig-proxy=true -u $(UID) --rm \
-	-e BUILD_WITH_CONTAINER="$(BUILD_WITH_CONTAINER)" \
 	-e TZ="$(TIMEZONE)" \
 	-e TARGET_ARCH="$(TARGET_ARCH)" \
 	-e TARGET_OS="$(TARGET_OS)" \
@@ -79,6 +80,7 @@ RUN = $(CONTAINER_CLI) run -t -i --sig-proxy=true -u $(UID) --rm \
 	--mount type=volume,source=home,destination="/home" \
 	-w /work $(IMG)
 else
+$(info Building with your local toolchain.)
 RUN =
 endif
 

Makefile.core.mk

@@ -329,7 +329,7 @@ fmt: format-go format-python
 
 # Build with -i to store the build caches into $GOPATH/pkg
 buildcache:
-	GOBUILDFLAGS=-i $(MAKE) build
+	GOBUILDFLAGS=-i $(MAKE) -f Makefile.core.mk build
 
 # List of all binaries to build
 BINARIES:=./istioctl/cmd/istioctl \
@@ -485,8 +485,8 @@ else
 endif
 test: | $(JUNIT_REPORT)
 	mkdir -p $(dir $(JUNIT_UNIT_TEST_XML))
-	KUBECONFIG="$${KUBECONFIG:-$${GO_TOP}/src/istio.io/istio/tests/util/kubeconfig}" \
-	$(MAKE) --keep-going $(TEST_OBJ) \
+	KUBECONFIG="$${KUBECONFIG:-$${REPO_ROOT}/tests/util/kubeconfig}" \
+	$(MAKE) -f Makefile.core.mk --keep-going $(TEST_OBJ) \
 	2>&1 | tee >($(JUNIT_REPORT) > $(JUNIT_UNIT_TEST_XML))
 
 GOTEST_PARALLEL ?= '-test.parallel=1'
@@ -578,7 +578,7 @@ common-coverage:
 RACE_TESTS ?= pilot-racetest mixer-racetest security-racetest galley-test common-racetest istioctl-racetest
 racetest: $(JUNIT_REPORT)
 	mkdir -p $(dir $(JUNIT_UNIT_TEST_XML))
-	$(MAKE) --keep-going $(RACE_TESTS) \
+	$(MAKE) -f Makefile.core.mk --keep-going $(RACE_TESTS) \
 	2>&1 | tee >($(JUNIT_REPORT) > $(JUNIT_UNIT_TEST_XML))
 
 .PHONY: pilot-racetest
@@ -640,7 +640,7 @@ gcs.push.deb: deb
 # generate_yaml in tests/istio.mk can build without specifying a hub & tag
 installgen:
 	install/updateVersion.sh -a ${HUB},${TAG}
-	$(MAKE) istio.yaml
+	$(MAKE) -f Makefile.core.mk istio.yaml
 
 $(HELM): $(ISTIO_OUT)
 	bin/init_helm.sh
@@ -693,7 +693,7 @@ generate_e2e_yaml: $(e2e_files)
 
 generate_e2e_yaml_coredump: export ENABLE_COREDUMP=true
 generate_e2e_yaml_coredump:
-	$(MAKE) generate_e2e_yaml
+	$(MAKE) -f Makefile.core.mk generate_e2e_yaml
 
 # Create yaml files for e2e tests. Applies values-e2e.yaml, then values-$filename.yaml
 $(e2e_files): $(HELM) $(HOME)/.helm istio-init.yaml

bin/init.sh

@@ -23,11 +23,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-# TODO(nmittler): Remove these variables and require that this script be run from the Makefile
-
-# Set GOPATH to match the expected layout
-GO_TOP=$(cd "$(dirname "$0")"/../../../..; pwd)
-
+export GO_TOP=${GO_TOP:-$(echo "${GOPATH}" | cut -d ':' -f1)}
 export OUT_DIR=${OUT_DIR:-${GO_TOP}/out}
 
 export GOPATH=${GOPATH:-$GO_TOP}

bin/init_helm.sh

@@ -22,11 +22,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-# TODO(nmittler): Remove these variables and require that this script be run from the Makefile
-
-# Set GOPATH to match the expected layout
-GO_TOP=$(cd "$(dirname "$0")"/../../../..; pwd)
-
+export GO_TOP=${GO_TOP:-$(echo "${GOPATH}" | cut -d ':' -f1)}
 export OUT_DIR=${OUT_DIR:-${GO_TOP}/out}
 
 HELM_VER=${HELM_VER:-v2.10.0}

bin/testEnvLocalK8S.sh

@@ -20,12 +20,11 @@ set -euo pipefail
 
 # Based on local kubeconfig - used to reproduce the environment and to improve local testing
 
-# expect istio scripts to be under $GOPATH/src/istio.io/istio/bin/...
-
 # If GOPATH is made up of several paths, use the first one for our targets in this file
 export GO_TOP=${GO_TOP:-$(echo "${GOPATH}" | cut -d ':' -f1)}
 
-export ISTIO_GO=${GO_TOP}/src/istio.io/istio
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export ISTIO_GO=${REPO_ROOT}
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
   export GOOS_LOCAL=darwin
@@ -39,15 +38,15 @@ export PATH=${GO_TOP}/bin:${PATH}
 export OUT=${GO_TOP}/out
 export ISTIO_OUT=${ISTIO_OUT:-${GO_TOP}/out/${GOOS_LOCAL}_amd64/release}
 
-# components used in the test (starting with circleci for consistency, eventually ci will use this)
+# components used in the test
 export K8S_VER=${K8S_VER:-v1.9.2}
 export ETCD_VER=${ETCD_VER:-v3.2.15}
 
 export MASTER_IP=127.0.0.1
 export MASTER_CLUSTER_IP=10.99.0.1
 
 # TODO: customize the ports and generate a local config
-export KUBECONFIG=${GO_TOP}/src/istio.io/istio/tests/util/kubeconfig
+export KUBECONFIG=${REPO_ROOT}/tests/util/kubeconfig
 
 "${ISTIO_GO}/bin/init.sh"
 

common/.commonfiles.sha

@@ -1 +1 @@
-b911e0db0ccb611ae4fd361d295af2ef5f8089d7
+db78a1cefc89e0941d93942f3850e0fad0b32b49

common/Makefile.common.mk

@@ -82,4 +82,4 @@ update-common-protos:
 	@cp -ar common-files/common-protos common-protos
 	@rm -fr common-files
 
-.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-pyhton lint-helm lint-markdown lint-sass lint-typescript lint-protos lint-all format-go format-python format-protos update-common update-common-protos
+.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-python lint-helm lint-markdown lint-sass lint-typescript lint-protos lint-all format-go format-python format-protos update-common update-common-protos

common/scripts/gobuild.sh

@@ -70,9 +70,22 @@ while read -r line; do
     LD_EXTRAFLAGS="${LD_EXTRAFLAGS} -X ${line}"
 done < "${BUILDINFO}"
 
+# verify go version before build
+# NB. this was copied verbatim from Kubernetes hack
+minimum_go_version=go1.13 # supported patterns: go1.x, go1.x.x (x should be a number)
+IFS=" " read -ra go_version <<< "$(${GOBINARY} version)"
+if [[ "${minimum_go_version}" != $(echo -e "${minimum_go_version}\n${go_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "${go_version[2]}" != "devel" ]]; then
+    echo "Warning: Detected that you are using an older version of the Go compiler. Istio requires ${minimum_go_version} or greater."
+fi
+
+OPTIMIZATION_FLAGS="-trimpath"
+if [ "${DEBUG}" == "1" ]; then
+    OPTIMIZATION_FLAGS=""
+fi
+
 time GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} ${GOBINARY} build \
         ${V} "${GOBUILDFLAGS_ARRAY[@]}" ${GCFLAGS:+-gcflags "${GCFLAGS}"} \
         -o "${OUT}" \
-        -trimpath \
+        ${OPTIMIZATION_FLAGS} \
         -pkgdir="${GOPKG}/${BUILD_GOOS}_${BUILD_GOARCH}" \
         -ldflags "${LDFLAGS} ${LD_EXTRAFLAGS}" "${@}"

prow/lib.sh

@@ -90,7 +90,7 @@ function clone_cni() {
   if [[ "$PWD" == "${GOPATH}/src/istio.io/istio" ]]; then
       TMP_DIR=$PWD
       cd ../ || return
-      git clone -b master "https://github.com/istio/cni.git"
+      git clone -b "${GIT_BRANCH}" "https://github.com/istio/cni.git"
       cd "${TMP_DIR}" || return
   fi
 }

release/gcb/gcb_lib.sh

@@ -26,7 +26,7 @@ source "${SCRIPTPATH}/docker_tag_push_lib.sh"
 function github_keys() {
   GITHUB_KEYFILE="${GITHUB_TOKEN_FILE}"
   export GITHUB_KEYFILE
-  
+
   if [[ -n "$CB_TEST_GITHUB_TOKEN_FILE_PATH" ]]; then
     local LOCAL_DIR
     LOCAL_DIR="$(mktemp -d /tmp/github.XXXX)"
@@ -72,13 +72,13 @@ function make_istio() {
   TAG=$4
   export TAG
   local BRANCH=$5
-  ISTIO_OUT=$(make DEBUG=0 where-is-out)
+  ISTIO_OUT=$(make -f Makefile.core.mk DEBUG=0 where-is-out)
   VERSION="${TAG}"
   export VERSION
   IFS='/' read -ra REPO <<< "$REL_DOCKER_HUB"
   MAKE_TARGETS=(istio-archive)
   MAKE_TARGETS+=(sidecar.deb)
-  
+
   CB_BRANCH=${BRANCH} DEBUG=0 ISTIO_DOCKER_HUB="${DOCKER_HUB}" HUB="${DOCKER_HUB}" make "${MAKE_TARGETS[@]}"
   mkdir -p "${OUTPUT_PATH}/deb"
   sha256sum "${ISTIO_OUT}/istio-sidecar.deb" > "${OUTPUT_PATH}/deb/istio-sidecar.deb.sha256"
@@ -88,7 +88,7 @@ function make_istio() {
   for file in "${ISTIO_OUT}"/archive/istioctl*.*; do
     sha256sum "${file}" > "$file.sha256"
   done
-  cp        "${ISTIO_OUT}"/archive/istioctl*.tar.gz "${OUTPUT_PATH}/"      
+  cp        "${ISTIO_OUT}"/archive/istioctl*.tar.gz "${OUTPUT_PATH}/"
   cp        "${ISTIO_OUT}"/archive/istioctl*.zip    "${OUTPUT_PATH}/"
   cp        "${ISTIO_OUT}"/archive/istioctl*.sha256 "${OUTPUT_PATH}/"
 
@@ -102,7 +102,7 @@ function make_istio() {
     pwd
     # tar the source code
     if [ -z "${LOCAL_BUILD+x}" ]; then
-      tar -czf "${OUTPUT_PATH}/source.tar.gz" go src --exclude go/out --exclude go/bin 
+      tar -czf "${OUTPUT_PATH}/source.tar.gz" go src --exclude go/out --exclude go/bin
     else
       tar -cvzf "${OUTPUT_PATH}/source.tar.gz" go/src/istio.io/istio go/src/istio.io/api go/src/istio.io/cni go/src/istio.io/proxy
     fi
@@ -112,14 +112,14 @@ function make_istio() {
 
   # log where git thinks the build might be dirty
   git status
-  
+
   pushd ../cni || exit
     #Handle CNI artifacts. Expects to be called from istio/cni repo.
-    CNI_OUT=$(make DEBUG=0 where-is-out)
+    CNI_OUT=$(make -f Makefile.core.mk DEBUG=0 where-is-out)
     rm -r "${CNI_OUT}/docker" || true
     # CNI version strategy is to have CNI run lock step with Istio i.e. CB_VERSION
     CB_BRANCH=${BRANCH} DEBUG=0 ISTIO_DOCKER_HUB="${DOCKER_HUB}" HUB="${DOCKER_HUB}" make build
-  
+
     CB_BRANCH=${BRANCH} DEBUG=0 ISTIO_DOCKER_HUB=${REL_DOCKER_HUB} HUB=${REL_DOCKER_HUB} make docker.save || exit 1
 
     cp -r "${CNI_OUT}/docker" "${OUTPUT_PATH}/"