Support pulling multi-arch envoy binaries (istio#39483)

* Allow no cpuinfo

* allow fail

* Bump kind image

* debug

* workaround env var

* fix docker build

* log

* arch in build

* set arch again

* fix arch type

* more logs

* more env

* Make single image architecture aware

* VM per-arch

* Opt out when requiring emulation

* Fix jwt server

* fmt

* Revert env var hacks

* cleanup

* minor fixes

* new release-builder

* multi-arch

* lint

Makefile.core.mk

@@ -146,6 +146,7 @@ default: init build test
 .PHONY: init
 # Downloads envoy, based on the SHA defined in the base pilot Dockerfile
 init: $(TARGET_OUT)/istio_is_init
+	echo ISTIO_ENVOY_LINUX_RELEASE_PATH=$(ISTIO_ENVOY_LINUX_RELEASE_PATH) TARGET_OUT_LINUX=$(TARGET_OUT_LINUX)
 	@mkdir -p ${TARGET_OUT}/logs
 	@mkdir -p ${TARGET_OUT}/release
 

bin/init.sh

@@ -26,6 +26,13 @@ if [[ "${TARGET_OUT_LINUX:-}" == "" ]]; then
   exit 1
 fi
 
+# Setup arch suffix for envoy binary. For backwards compatibility, amd64 has no suffix.
+if [[ "${TARGET_ARCH}" == "amd64" ]]; then
+	ISTIO_ENVOY_ARCH_SUFFIX=""
+else
+	ISTIO_ENVOY_ARCH_SUFFIX="-${TARGET_ARCH}"
+fi
+
 # Populate the git version for istio/proxy (i.e. Envoy)
 PROXY_REPO_SHA="${PROXY_REPO_SHA:-$(grep PROXY_REPO_SHA istio.deps  -A 4 | grep lastStableSHA | cut -f 4 -d '"')}"
 
@@ -36,9 +43,9 @@ SIDECAR="${SIDECAR:-envoy}"
 
 # OS-neutral vars. These currently only work for linux.
 ISTIO_ENVOY_VERSION="${ISTIO_ENVOY_VERSION:-${PROXY_REPO_SHA}}"
-ISTIO_ENVOY_DEBUG_URL="${ISTIO_ENVOY_DEBUG_URL:-${ISTIO_ENVOY_BASE_URL}/envoy-debug-${ISTIO_ENVOY_VERSION}.tar.gz}"
+ISTIO_ENVOY_DEBUG_URL="${ISTIO_ENVOY_DEBUG_URL:-${ISTIO_ENVOY_BASE_URL}/envoy-debug-${ISTIO_ENVOY_VERSION}${ISTIO_ENVOY_ARCH_SUFFIX}.tar.gz}"
 ISTIO_ENVOY_CENTOS_DEBUG_URL="${ISTIO_ENVOY_CENTOS_DEBUG_URL:-${ISTIO_ENVOY_BASE_URL}/envoy-centos-debug-${ISTIO_ENVOY_VERSION}.tar.gz}"
-ISTIO_ENVOY_RELEASE_URL="${ISTIO_ENVOY_RELEASE_URL:-${ISTIO_ENVOY_BASE_URL}/envoy-alpha-${ISTIO_ENVOY_VERSION}.tar.gz}"
+ISTIO_ENVOY_RELEASE_URL="${ISTIO_ENVOY_RELEASE_URL:-${ISTIO_ENVOY_BASE_URL}/envoy-alpha-${ISTIO_ENVOY_VERSION}${ISTIO_ENVOY_ARCH_SUFFIX}.tar.gz}"
 ISTIO_ENVOY_CENTOS_RELEASE_URL="${ISTIO_ENVOY_CENTOS_RELEASE_URL:-${ISTIO_ENVOY_BASE_URL}/envoy-centos-alpha-${ISTIO_ENVOY_VERSION}.tar.gz}"
 
 # Envoy Linux vars.

pkg/test/echo/docker/Dockerfile.app_sidecar

@@ -8,15 +8,15 @@ COPY certs/                           /var/lib/istio/
 COPY certs/default/                   /var/run/secrets/istio/
 
 # Install the sidecar components
-COPY istio-sidecar.deb  /tmp/istio-sidecar.deb
+ARG TARGETARCH
+COPY ${TARGETARCH:-amd64}/istio-sidecar.deb  /tmp/istio-sidecar.deb
 RUN dpkg -i /tmp/istio-sidecar.deb && rm /tmp/istio-sidecar.deb
 
 # Sudoers used to allow tcpdump and other debug utilities.
 COPY sudoers /etc/sudoers
 
 # Install the Echo application
 COPY echo-start.sh /usr/local/bin/echo-start.sh
-ARG TARGETARCH
 COPY ${TARGETARCH:-amd64}/client /usr/local/bin/client
 COPY ${TARGETARCH:-amd64}/server /usr/local/bin/server
 

prow/integ-suite-kind.sh

@@ -37,7 +37,7 @@ setup_and_export_git_sha
 source "${ROOT}/common/scripts/kind_provisioner.sh"
 
 TOPOLOGY=SINGLE_CLUSTER
-NODE_IMAGE="gcr.io/istio-testing/kind-node:v1.24.0-0.13.0"
+NODE_IMAGE="gcr.io/istio-testing/kind-node:v1.24.3"
 KIND_CONFIG=""
 CLUSTER_TOPOLOGY_CONFIG_FILE="${ROOT}/prow/config/topology/multicluster.json"
 
@@ -106,7 +106,7 @@ done
 
 if [ -f /proc/cpuinfo ]; then
   echo "Checking CPU..."
-  grep 'model' /proc/cpuinfo
+  grep 'model' /proc/cpuinfo || true
 fi
 
 # Default IP family of the cluster is IPv4

prow/lib.sh

@@ -125,11 +125,16 @@ function build_images() {
     targets+="docker.operator "
   fi
   targets+="docker.install-cni "
+  # Integration tests are always running on local architecture (no cross compiling), so find out what that is.
+  arch="linux/amd64"
+  if [[ "$(uname -m)" == "aarch64" ]]; then
+      arch="linux/arm64"
+  fi
   if [[ "${VARIANT:-default}" == "distroless" ]]; then
-    DOCKER_BUILD_VARIANTS="distroless" DOCKER_TARGETS="${targets}" make dockerx.pushx
-    DOCKER_BUILD_VARIANTS="default" DOCKER_TARGETS="${nonDistrolessTargets}" make dockerx.pushx
+    DOCKER_ARCHITECTURES="${arch}" DOCKER_BUILD_VARIANTS="distroless" DOCKER_TARGETS="${targets}" make dockerx.pushx
+    DOCKER_ARCHITECTURES="${arch}" DOCKER_BUILD_VARIANTS="default" DOCKER_TARGETS="${nonDistrolessTargets}" make dockerx.pushx
   else
-    DOCKER_BUILD_VARIANTS="${VARIANT:-default}" DOCKER_TARGETS="${targets} ${nonDistrolessTargets}" make dockerx.pushx
+   DOCKER_ARCHITECTURES="${arch}"  DOCKER_BUILD_VARIANTS="${VARIANT:-default}" DOCKER_TARGETS="${targets} ${nonDistrolessTargets}" make dockerx.pushx
   fi
 }
 

prow/release-commit.sh

@@ -32,7 +32,7 @@ DOCKER_HUB=${DOCKER_HUB:-gcr.io/istio-testing}
 GCS_BUCKET=${GCS_BUCKET:-istio-build/dev}
 
 # Use a pinned version in case breaking changes are needed
-BUILDER_SHA=49c3b9df16351ca44455c8a12be5829c7eef3a25
+BUILDER_SHA=55c3306e83172818e87af7911756c249f142a4b1
 
 # Reference to the next minor version of Istio
 # This will create a version like 1.4-alpha.sha
@@ -75,6 +75,7 @@ ${DEPENDENCIES:-$(cat <<EOD
   release-builder:
     git: https://github.com/istio/release-builder
     sha: ${BUILDER_SHA}
+architectures: [linux/amd64, linux/arm64]
 EOD
 )}
 dashboards:

samples/jwt-server/jwt-server.yaml

@@ -106,7 +106,7 @@ spec:
         app: jwt-server
     spec:
       containers:
-      - image: gcr.io/istio-testing/jwt-server:0.6
+      - image: gcr.io/istio-testing/jwt-server:0.7
         imagePullPolicy: IfNotPresent
         name: jwt-server
         volumeMounts:

samples/jwt-server/src/Dockerfile

@@ -13,12 +13,12 @@
 #   limitations under the License.
 
 # build a jwt-server binary using the golang container
-FROM golang:1.15 as builder
+FROM golang:1.19 as builder
 WORKDIR /go/src/istio.io/jwt-server/
 COPY . .
 RUN CGO_ENABLED=0 GOOS=linux go build -o jwt-server main.go
 
-FROM gcr.io/distroless/static-debian10@sha256:4433370ec2b3b97b338674b4de5ffaef8ce5a38d1c9c0cb82403304b8718cde9 as distroless
+FROM gcr.io/distroless/static-debian11@sha256:21d3f84a4f37c36199fd07ad5544dcafecc17776e3f3628baf9a57c8c0181b3f as distroless
 
 WORKDIR /bin/
 # copy the jwt-server binary to a separate container based on BASE_DISTRIBUTION

samples/jwt-server/src/Makefile

@@ -13,10 +13,7 @@
 # limitations under the License.
 
 HUB ?= gcr.io/istio-testing/jwt-server
-TAG ?= 0.6
+TAG ?= 0.7
 
-build: main.go Dockerfile
-	docker build . -t $(HUB):$(TAG)
-
-push: build
-	docker push $(HUB):$(TAG)
+push: main.go Dockerfile
+	docker buildx build --push --platform=linux/amd64,linux/arm64 . -t $(HUB):$(TAG)

tools/docker-builder/builder/crane.go

@@ -31,6 +31,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 
 	"istio.io/pkg/log"
 )
@@ -65,35 +66,50 @@ type Args struct {
 	FilesBase string
 }
 
+type baseKey struct {
+	arch string
+	name string
+}
+
 var (
-	bases   = map[string]v1.Image{}
+	bases   = map[baseKey]v1.Image{}
 	basesMu sync.RWMutex
 )
 
-func WarmBase(baseImages ...string) {
+func WarmBase(architectures []string, baseImages ...string) {
 	basesMu.Lock()
 	wg := sync.WaitGroup{}
-	wg.Add(len(baseImages))
-	resolvedBaseImages := make([]v1.Image, len(baseImages))
+	wg.Add(len(baseImages) * len(architectures))
+	resolvedBaseImages := make([]v1.Image, len(baseImages)*len(architectures))
+	keys := []baseKey{}
+	for _, a := range architectures {
+		for _, b := range baseImages {
+			keys = append(keys, baseKey{toPlatform(a).Architecture, b})
+		}
+	}
 	go func() {
 		wg.Wait()
 		for i, rbi := range resolvedBaseImages {
-			bases[baseImages[i]] = rbi
+			bases[keys[i]] = rbi
 		}
 		basesMu.Unlock()
 	}()
 
 	t0 := time.Now()
-	for i, b := range baseImages {
+	for i, b := range keys {
 		b, i := b, i
 		go func() {
 			defer wg.Done()
-			ref, err := name.ParseReference(b)
+			ref, err := name.ParseReference(b.name)
 			if err != nil {
 				log.WithLabels("image", b).Warnf("base failed: %v", err)
 				return
 			}
-			bi, err := remote.Image(ref, remote.WithProgress(CreateProgress(fmt.Sprintf("base %v", ref))))
+			plat := v1.Platform{
+				Architecture: b.arch,
+				OS:           "linux",
+			}
+			bi, err := remote.Image(ref, remote.WithPlatform(plat), remote.WithProgress(CreateProgress(fmt.Sprintf("base %v", ref))))
 			if err != nil {
 				log.WithLabels("image", b).Warnf("base failed: %v", err)
 				return
@@ -140,10 +156,11 @@ func Build(b BuildSpec) error {
 
 	var images []v1.Image
 	for _, args := range b.Args {
+		plat := toPlatform(args.Arch)
 		baseImage := empty.Image
 		if args.Base != "" {
 			basesMu.RLock()
-			baseImage = bases[args.Base]
+			baseImage = bases[baseKey{arch: plat.Architecture, name: args.Base}] // todo per-arch base
 			basesMu.RUnlock()
 		}
 		if baseImage == nil {
@@ -152,7 +169,11 @@ func Build(b BuildSpec) error {
 			if err != nil {
 				return err
 			}
-			bi, err := remote.Image(ref, remote.WithProgress(CreateProgress(fmt.Sprintf("base %v", ref))))
+			bi, err := remote.Image(
+				ref,
+				remote.WithPlatform(plat),
+				remote.WithProgress(CreateProgress(fmt.Sprintf("base %v", ref))),
+			)
 			if err != nil {
 				return err
 			}
@@ -167,6 +188,11 @@ func Build(b BuildSpec) error {
 
 		trace("base config")
 
+		// Set our platform on the image. This is largely for empty.Image only; others should already have correct default.
+		cfgFile = cfgFile.DeepCopy()
+		cfgFile.OS = plat.OS
+		cfgFile.Architecture = plat.Architecture
+
 		cfg := cfgFile.Config
 		for k, v := range args.Env {
 			cfg.Env = append(cfg.Env, fmt.Sprintf("%v=%v", k, v))
@@ -232,6 +258,7 @@ func Build(b BuildSpec) error {
 	} else {
 		// Multiple, we need to create an index
 		var manifest v1.ImageIndex = empty.Index
+		manifest = mutate.IndexMediaType(manifest, types.DockerManifestList)
 		for idx, i := range images {
 			img := i
 			mt, err := img.MediaType()
@@ -248,18 +275,14 @@ func Build(b BuildSpec) error {
 			if err != nil {
 				return fmt.Errorf("failed to compute size: %w", err)
 			}
-			os, arch, _ := strings.Cut(b.Args[idx].Arch, "/")
+			plat := toPlatform(b.Args[idx].Arch)
 			manifest = mutate.AppendManifests(manifest, mutate.IndexAddendum{
 				Add: i,
 				Descriptor: v1.Descriptor{
 					MediaType: mt,
 					Size:      size,
 					Digest:    h,
-					Platform: &v1.Platform{
-						Architecture: arch,
-						OS:           os,
-						Variant:      "", // TODO?
-					},
+					Platform:  &plat,
 				},
 			})
 		}
@@ -299,6 +322,15 @@ func Build(b BuildSpec) error {
 	return nil
 }
 
+func toPlatform(archString string) v1.Platform {
+	os, arch, _ := strings.Cut(archString, "/")
+	return v1.Platform{
+		Architecture: arch,
+		OS:           os,
+		Variant:      "", // TODO?
+	}
+}
+
 func CreateProgress(name string) chan v1.Update {
 	updates := make(chan v1.Update, 1000)
 	go func() {

tools/docker-builder/common.go

@@ -25,13 +25,13 @@ func extractTags(a Args, target, variant string, hasDoubleDefault bool) []string
 		for _, tg := range a.Tags {
 			if variant == DefaultVariant {
 				// For default, we have no suffix
-				tags = append(tags, fmt.Sprintf("%s/%s:%s", h, target, tg))
+				tags = append(tags, fmt.Sprintf("%s/%s:%s%s", h, target, tg, a.suffix))
 			} else {
 				// Otherwise, we have a suffix with the variant
-				tags = append(tags, fmt.Sprintf("%s/%s:%s-%s", h, target, tg, variant))
+				tags = append(tags, fmt.Sprintf("%s/%s:%s-%s%s", h, target, tg, variant, a.suffix))
 				// If we need a default as well, add it as a second tag for the same image to avoid building twice
 				if variant == PrimaryVariant && hasDoubleDefault {
-					tags = append(tags, fmt.Sprintf("%s/%s:%s", h, target, tg))
+					tags = append(tags, fmt.Sprintf("%s/%s:%s%s", h, target, tg, a.suffix))
 				}
 			}
 		}

tools/docker-builder/crane.go

@@ -65,7 +65,11 @@ func RunCrane(a Args) error {
 				Dests: extractTags(a, t, v, hasDoubleDefault),
 			}
 			for _, arch := range a.Architectures {
-				df := a.PlanFor(arch).Find(t).Dockerfile
+				p := a.PlanFor(arch).Find(t)
+				if p == nil {
+					continue
+				}
+				df := p.Dockerfile
 				dargs := createArgs(a, t, v, arch)
 				args, err := dockerfile.Parse(df, dockerfile.WithArgs(dargs), dockerfile.IgnoreRuns())
 				if err != nil {
@@ -76,6 +80,9 @@ func RunCrane(a Args) error {
 				// args.Files provides a mapping from final destination -> docker context source
 				// docker context is virtual, so we need to rewrite the "docker context source" to the real path of disk
 				plan := a.PlanFor(arch).Find(args.Name)
+				if plan == nil {
+					continue
+				}
 				// Plan is a list of real file paths, but we don't have a strong mapping from "docker context source"
 				// to "real path on disk". We do have a weak mapping though, by reproducing docker-copy.sh
 				for dest, src := range args.Files {
@@ -94,7 +101,7 @@ func RunCrane(a Args) error {
 
 	// Warm up our base images while we are building everything. This isn't pulling them -- we actually
 	// never pull them -- but it is pulling the config file from the remote registry.
-	builder.WarmBase(bases.SortedList()...)
+	builder.WarmBase(a.Architectures, bases.SortedList()...)
 
 	// Build all dependencies
 	makeStart := time.Now()