envtest: add option to download binaries, bump envtest to v1.32.0

Signed-off-by: Stefan Büringer buringerst@vmware.com

Author: sbueringer

hack/check-everything.sh

@@ -30,7 +30,7 @@ export GOTOOLCHAIN="go$(make --silent go-version)"
 ${hack_dir}/verify.sh
 
 # Envtest.
-ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.28.0"}
+ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.32.0"}
 
 header_text "installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest if necessary"
 tmp_bin=/tmp/cr-tests-bin

pkg/client/apiutil/restmapper_test.go

@@ -77,6 +77,8 @@ func setupEnvtest(t *testing.T, disableAggregatedDiscovery bool) *rest.Config {
 		CRDDirectoryPaths: []string{"testdata"},
 	}
 	if disableAggregatedDiscovery {
+		testEnv.DownloadBinaryAssets = true
+		testEnv.DownloadBinaryAssetsVersion = "v1.28.0"
 		testEnv.ControlPlane.GetAPIServer().Configure().Append("feature-gates", "AggregatedDiscoveryEndpoint=false")
 	}
 

pkg/envtest/binaries.go

@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2021 The Kubernetes Authors
+
+package envtest
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha512"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+
+	"sigs.k8s.io/yaml"
+)
+
+// DefaultIndexURL is the default index used in HTTPClient.
+var DefaultIndexURL = "https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml"
+
+// Index represents an index of envtest binary archives. Example:
+//
+//	releases:
+//		v1.28.0:
+//			envtest-v1.28.0-darwin-amd64.tar.gz:
+//	    		hash: <sha512-hash>
+//				selfLink: <url-to-archive-with-envtest-binaries>
+type Index struct {
+	// Releases maps Kubernetes versions to Releases (envtest archives).
+	Releases map[string]Release `json:"releases"`
+}
+
+// Release maps an archive name to an archive.
+type Release map[string]Archive
+
+// Archive contains the self link to an archive and its hash.
+type Archive struct {
+	Hash     string `json:"hash"`
+	SelfLink string `json:"selfLink"`
+}
+
+func downloadBinaryAssets(ctx context.Context, binaryAssetsDirectory, binaryAssetsVersion string) (string, string, string, error) {
+	var downloadDir string
+	if binaryAssetsDirectory != "" {
+		downloadDir = binaryAssetsDirectory
+		if !fileExists(downloadDir) {
+			if err := os.Mkdir(binaryAssetsDirectory, 0700); err != nil {
+				return "", "", "", fmt.Errorf("failed to create dir for envtest binaries %q: %w", binaryAssetsDirectory, err)
+			}
+		}
+	} else {
+		var err error
+		if downloadDir, err = os.MkdirTemp("", "envtest-binaries-"); err != nil {
+			return "", "", "", fmt.Errorf("failed to create tmp dir for envtest binaries: %w", err)
+		}
+	}
+
+	apiServerPath := path.Join(downloadDir, "kube-apiserver")
+	etcdPath := path.Join(downloadDir, "etcd")
+	kubectlPath := path.Join(downloadDir, "kubectl")
+
+	if fileExists(apiServerPath) && fileExists(etcdPath) && fileExists(kubectlPath) {
+		return apiServerPath, etcdPath, kubectlPath, nil
+	}
+
+	buf := &bytes.Buffer{}
+	if err := DownloadBinaryAssets(ctx, binaryAssetsVersion, buf); err != nil {
+		return "", "", "", fmt.Errorf("failed to create tmp file to download envtest binaries: %w", err)
+	}
+
+	gzStream, err := gzip.NewReader(buf)
+	if err != nil {
+		return "", "", "", fmt.Errorf("failed to read TODO: %w", err)
+	}
+	tarReader := tar.NewReader(gzStream)
+
+	var header *tar.Header
+	for header, err = tarReader.Next(); err == nil; header, err = tarReader.Next() {
+		if header.Typeflag != tar.TypeReg { // Skipping non-regular file entry in archive
+			continue
+		}
+
+		// just dump all files to the main path, ignoring the prefixed directory
+		// paths -- they're redundant.  We also ignore bits for the most part (except for X),
+		// preferfing our own scheme.
+		fileName := filepath.Base(header.Name)
+
+		perms := 0555 & header.Mode // make sure we're at most r+x
+
+		binOut, err := os.OpenFile(path.Join(downloadDir, fileName), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_TRUNC, os.FileMode(perms))
+		if err != nil {
+			if os.IsExist(err) {
+				continue
+			}
+			return "", "", "", fmt.Errorf("unable to create file %s from archive to disk for version-platform pair %s: %w", fileName, downloadDir, err)
+		}
+		if err := func() error {
+			defer binOut.Close()
+			if _, err := io.Copy(binOut, tarReader); err != nil {
+				return fmt.Errorf("unable to write file %s from archive to disk for version-platform pair %s", fileName, downloadDir)
+			}
+			return nil
+		}(); err != nil {
+			return "", "", "", err
+		}
+	}
+
+	return apiServerPath, etcdPath, kubectlPath, nil
+}
+
+func fileExists(path string) bool {
+	if _, err := os.Stat(path); err == nil {
+		return true
+	}
+	return false
+}
+
+// DownloadBinaryAssets downloads the given concrete version for the given concrete platform, writing it to the out.
+func DownloadBinaryAssets(ctx context.Context, version string, out io.Writer) error {
+	index, err := getIndex(ctx)
+	if err != nil {
+		return err
+	}
+
+	var loc *url.URL
+	var name string
+
+	archives, ok := index.Releases[version]
+	if !ok {
+		return fmt.Errorf("error finding binaries for version %s", version)
+	}
+
+	archiveName := fmt.Sprintf("envtest-%s-%s-%s.tar.gz", version, runtime.GOOS, runtime.GOARCH)
+	archive, ok := archives[archiveName]
+	if !ok {
+		return fmt.Errorf("error finding binaries for version %s with archiveName %s", version, archiveName)
+	}
+
+	loc, err = url.Parse(archive.SelfLink)
+	if err != nil {
+		return fmt.Errorf("error parsing selfLink %q, %w", loc, err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil)
+	if err != nil {
+		return fmt.Errorf("unable to construct request to fetch %s: %w", name, err)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("unable to fetch %s (%s): %w", name, req.URL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("unable fetch %s (%s) -- got status %q", name, req.URL, resp.Status)
+	}
+
+	return readBody(resp, out, name, archive.Hash)
+}
+
+func getIndex(ctx context.Context) (*Index, error) {
+	loc, err := url.Parse(DefaultIndexURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse index URL: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("unable to construct request to get index: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("unable to perform request to get index: %w", err)
+	}
+
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("unable to get index -- got status %q", resp.Status)
+	}
+
+	responseBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get index -- unable to read body %w", err)
+	}
+
+	var index Index
+	if err := yaml.Unmarshal(responseBody, &index); err != nil {
+		return nil, fmt.Errorf("unable to unmarshal index: %w", err)
+	}
+	return &index, nil
+}
+
+func readBody(resp *http.Response, out io.Writer, archiveName string, expectedHash string) error {
+	// stream in chunks to do the checksum, don't load the whole thing into
+	// memory to avoid causing issues with big files.
+	buf := make([]byte, 32*1024) // 32KiB, same as io.Copy
+	hasher := sha512.New()
+
+	for cont := true; cont; {
+		amt, err := resp.Body.Read(buf)
+		if err != nil && !errors.Is(err, io.EOF) {
+			return fmt.Errorf("unable read next chunk of %s: %w", archiveName, err)
+		}
+		if amt > 0 {
+			// checksum never returns errors according to docs
+			hasher.Write(buf[:amt])
+			if _, err := out.Write(buf[:amt]); err != nil {
+				return fmt.Errorf("unable write next chunk of %s: %w", archiveName, err)
+			}
+		}
+		cont = amt > 0 && !errors.Is(err, io.EOF)
+	}
+
+	actualHash := hex.EncodeToString(hasher.Sum(nil))
+	if actualHash != expectedHash {
+		return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (reported)", archiveName, actualHash, expectedHash)
+	}
+
+	return nil
+}

pkg/envtest/server.go

@@ -147,6 +147,12 @@ type Environment struct {
 	// values are merged.
 	CRDDirectoryPaths []string
 
+	// DownloadBinaryAssets indicates that the binaries should be downloaded. //FIXME: doc
+	DownloadBinaryAssets bool
+
+	// DownloadBinaryAssetsVersion is the version of binaries to download. //FIXME: doc
+	DownloadBinaryAssetsVersion string
+
 	// BinaryAssetsDirectory is the path where the binaries required for the envtest are
 	// located in the local environment. This field can be overridden by setting KUBEBUILDER_ASSETS.
 	BinaryAssetsDirectory string
@@ -233,9 +239,20 @@ func (te *Environment) Start() (*rest.Config, error) {
 			}
 		}
 
-		apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory)
-		te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory)
-		te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory)
+		if te.DownloadBinaryAssets {
+			apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(context.TODO(), te.BinaryAssetsDirectory, te.DownloadBinaryAssetsVersion)
+			if err != nil {
+				return nil, err
+			}
+
+			apiServer.Path = apiServerPath
+			te.ControlPlane.Etcd.Path = etcdPath
+			te.ControlPlane.KubectlPath = kubectlPath
+		} else {
+			apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory)
+			te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory)
+			te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory)
+		}
 
 		if err := te.defaultTimeouts(); err != nil {
 			return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err)