Replace grab with internal download function

go.mod

@@ -13,7 +13,6 @@ require (
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/bombsimon/logrusr/v4 v4.1.0
 	github.com/carlmjohnson/requests v0.24.2
-	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/cilium/ebpf v0.16.0
 	github.com/cloudflare/cfssl v1.6.4
 	github.com/containerd/cgroups/v3 v3.0.3
@@ -56,6 +55,7 @@ require (
 	golang.org/x/mod v0.21.0
 	golang.org/x/sync v0.8.0
 	golang.org/x/sys v0.25.0
+	golang.org/x/text v0.18.0
 	golang.org/x/tools v0.25.0
 	google.golang.org/grpc v1.67.0
 	helm.sh/helm/v3 v3.16.1
@@ -260,7 +260,6 @@ require (
 	golang.org/x/net v0.29.0 // indirect
 	golang.org/x/oauth2 v0.22.0 // indirect
 	golang.org/x/term v0.24.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect

go.sum

@@ -69,8 +69,6 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXe
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/carlmjohnson/requests v0.24.2 h1:JDakhAmTIKL/qL/1P7Kkc2INGBJIkIFP6xUeUmPzLso=
 github.com/carlmjohnson/requests v0.24.2/go.mod h1:duYA/jDnyZ6f3xbcF5PpZ9N8clgopubP2nK5i6MVMhU=
-github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
-github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

internal/http/download.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package http
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	internalio "github.com/k0sproject/k0s/internal/io"
+	"github.com/k0sproject/k0s/pkg/build"
+	"github.com/k0sproject/k0s/pkg/k0scontext"
+)
+
+type DownloadOption func(*downloadOptions)
+
+// Downloads the contents of the given URL. Writes the HTTP response body to writer.
+// Stalled downloads will be aborted if there's no data transfer for some time.
+func Download(ctx context.Context, url string, target io.Writer, options ...DownloadOption) (err error) {
+	opts := downloadOptions{
+		stalenessTimeout: time.Minute,
+	}
+	for _, opt := range options {
+		opt(&opts)
+	}
+
+	// Prepare the client and the request.
+	client := http.Client{
+		Transport: &http.Transport{
+			// This is a one-shot HTTP client which should release resources immediately.
+			DisableKeepAlives: true,
+		},
+	}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return fmt.Errorf("invalid download request: %w", err)
+	}
+	req.Header.Set("User-Agent", "k0s/"+build.Version)
+
+	// Create a context with an inactivity timeout to cancel the download if it stalls.
+	ctx, cancel, keepAlive := k0scontext.WithInactivityTimeout(ctx, opts.stalenessTimeout)
+	defer cancel(nil)
+
+	// Execute the request.
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		if cause := context.Cause(ctx); cause != nil && !errors.Is(err, cause) {
+			err = fmt.Errorf("%w (%w)", cause, err)
+		}
+		return fmt.Errorf("request failed: %w", err)
+	}
+	defer func() {
+		if closeErr := resp.Body.Close(); closeErr != nil {
+			err = errors.Join(err, closeErr)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	if err := opts.detectRemoteFileName(resp); err != nil {
+		return err
+	}
+
+	// Monitor writes. Keep the download context alive as long as data is flowing.
+	writeMonitor := internalio.WriterFunc(func(p []byte) (int, error) {
+		len := len(p)
+		if len > 0 {
+			keepAlive()
+		}
+		return len, nil
+	})
+
+	// Run the actual data transfer.
+	if _, err := io.Copy(io.MultiWriter(writeMonitor, target), resp.Body); err != nil {
+		if cause := context.Cause(ctx); cause != nil && !errors.Is(err, cause) {
+			err = fmt.Errorf("%w (%w)", cause, err)
+		}
+
+		return fmt.Errorf("while downloading: %w", err)
+	}
+
+	return nil
+}
+
+// Sets the staleness timeout for a download.
+// Defaults to one minute if omitted.
+func WithStalenessTimeout(stalenessTimeout time.Duration) DownloadOption {
+	return func(opts *downloadOptions) {
+		opts.stalenessTimeout = stalenessTimeout
+	}
+}
+
+type downloadOptions struct {
+	stalenessTimeout time.Duration
+	downloadFileNameOptions
+}

internal/http/download_test.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package http_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	internalhttp "github.com/k0sproject/k0s/internal/http"
+	internalio "github.com/k0sproject/k0s/internal/io"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownload_CancelRequest(t *testing.T) {
+	ctx, cancel := context.WithCancelCause(context.TODO())
+	cancel(assert.AnError)
+
+	err := internalhttp.Download(ctx, "http://404.example.com", io.Discard)
+	assert.ErrorIs(t, err, assert.AnError)
+	if urlErr := (*url.Error)(nil); assert.ErrorAs(t, err, &urlErr) {
+		assert.Equal(t, "Get", urlErr.Op)
+		assert.Equal(t, "http://404.example.com", urlErr.URL)
+		assert.Equal(t, context.Canceled, urlErr.Err)
+	}
+}
+
+func TestDownload_NoContent(t *testing.T) {
+	baseURL := startFakeDownloadServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	err := internalhttp.Download(context.TODO(), baseURL, io.Discard)
+	assert.ErrorContains(t, err, "bad status: 204 No Content")
+}
+
+func TestDownload_ShortDownload(t *testing.T) {
+	baseURL := startFakeDownloadServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("Content-Length", "10")
+		_, err := w.Write([]byte("too short")) // this is only 9 bytes
+		assert.NoError(t, err)
+	}))
+
+	err := internalhttp.Download(context.TODO(), baseURL, io.Discard)
+	assert.ErrorContains(t, err, "while downloading: unexpected EOF")
+}
+
+func TestDownload_ExcessContentLength(t *testing.T) {
+	baseURL := startFakeDownloadServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("Content-Length", "4")
+		_, err := w.Write([]byte("yolo"))
+		assert.NoError(t, err)
+		// Excess content length
+		_, err = w.Write([]byte("<-stripped"))
+		assert.ErrorIs(t, err, http.ErrContentLength)
+	}))
+
+	var downloaded strings.Builder
+	err := internalhttp.Download(context.TODO(), baseURL, &downloaded)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "yolo", downloaded.String())
+}
+
+func TestDownload_CancelDownload(t *testing.T) {
+	ctx, cancel := context.WithCancelCause(context.TODO())
+	t.Cleanup(func() { cancel(nil) })
+
+	baseURL := startFakeDownloadServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for {
+			if _, err := w.Write([]byte(t.Name())); !assert.NoError(t, err) {
+				return
+			}
+
+			select {
+			case <-r.Context().Done():
+				return
+			case <-time.After(time.Microsecond):
+			}
+		}
+	}))
+
+	err := internalhttp.Download(ctx, baseURL, internalio.WriterFunc(func(p []byte) (int, error) {
+		cancel(assert.AnError)
+		return len(p), nil
+	}))
+
+	assert.ErrorContains(t, err, "while downloading: ")
+	assert.ErrorIs(t, err, assert.AnError)
+	assert.ErrorIs(t, err, context.Canceled)
+}
+
+func TestDownload_RedirectLoop(t *testing.T) {
+	// The current implementation doesn't detect loops, but it stops after 10 redirects.
+
+	expectedRequests := uint32(10)
+	var requests atomic.Uint32
+	var baseURL string
+	baseURL = startFakeDownloadServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !assert.LessOrEqual(t, requests.Add(1), expectedRequests, "More requests than anticipated") {
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+
+		// Produce redirect loops: /looper-0, /looper-1, /looper-2, /looper-0, ...
+		var seq uint8
+		if _, lastSeq, found := strings.Cut(r.URL.Path, "/looper-"); found {
+			if lastSeq, err := strconv.ParseUint(lastSeq, 10, 4); err == nil && lastSeq < 3 {
+				seq = uint8(lastSeq) + 1
+			}
+		}
+
+		http.Redirect(w, r, fmt.Sprintf("%s/looper-%d", baseURL, seq), http.StatusSeeOther)
+	}))
+
+	var downloaded strings.Builder
+	err := internalhttp.Download(context.TODO(), baseURL, &downloaded)
+
+	assert.Equal(t, expectedRequests, requests.Load())
+	assert.ErrorContains(t, err, "stopped after 10 redirects")
+}
+
+func startFakeDownloadServer(t *testing.T, handler http.Handler) string {
+	server := &http.Server{Addr: "localhost:0", Handler: handler}
+	listener, err := net.Listen("tcp", server.Addr)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	serverError := make(chan error)
+	go func() {
+		defer close(serverError)
+		serverError <- server.Serve(listener)
+	}()
+
+	t.Cleanup(func() {
+		err := server.Shutdown(context.Background())
+		if !assert.NoError(t, err, "Couldn't shutdown HTTP server") {
+			return
+		}
+
+		assert.ErrorIs(t, <-serverError, http.ErrServerClosed, "HTTP server terminated unexpectedly")
+	})
+
+	return (&url.URL{Scheme: "http", Host: listener.Addr().String()}).String()
+}