Merge pull request #1823 from AkihiroSuda/buildkit-0.11

update BuildKit (0.11.0) ; nerdctl system prune: drop Go dependency on BuildKit

Dockerfile

@@ -23,7 +23,7 @@ ARG RUNC_VERSION=v1.1.4
 ARG CNI_PLUGINS_VERSION=v1.1.1
 
 # Extra deps: Build
-ARG BUILDKIT_VERSION=v0.10.6
+ARG BUILDKIT_VERSION=v0.11.0
 # Extra deps: Lazy-pulling
 ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0
 # Extra deps: Nydus Lazy-pulling

Dockerfile.d/SHA256SUMS.d/buildkit-v0.11.0

@@ -0,0 +1,2 @@
+d60fe269b03e6ecf73ea4525b9d9ae2734e0846d55a037bc03eb570a1571e63f  buildkit-v0.11.0.linux-amd64.tar.gz
+ea1f10c7a7f0ddab94ba840fb24d30fa2f5093cc3f822d24e3ee7e324a566e2d  buildkit-v0.11.0.linux-arm64.tar.gz

README.md

@@ -101,6 +101,7 @@ In addition to containerd, the following components should be installed:
 - [CNI plugins](https://github.com/containernetworking/plugins): for using `nerdctl run`.
    - v1.1.0 or later is highly recommended. Older versions require extra [CNI isolation plugin](https://github.com/AkihiroSuda/cni-isolation) for isolating bridge networks (`nerdctl network create`).
 - [BuildKit](https://github.com/moby/buildkit) (OPTIONAL): for using `nerdctl build`. BuildKit daemon (`buildkitd`) needs to be running. See also [the document about setting up BuildKit](./docs/build.md).
+   - v0.11.0 or later is highly recommended. Some features, such as pruning caches with `nerdctl system prune`, do not work with older versions.
 - [RootlessKit](https://github.com/rootless-containers/rootlesskit) and [slirp4netns](https://github.com/rootless-containers/slirp4netns) (OPTIONAL): for [Rootless mode](./docs/rootless.md)
    - RootlessKit needs to be v0.10.0 or later. v0.14.1 or later is recommended.
    - slirp4netns needs to be v0.4.0 or later. v1.1.7 or later is recommended.

cmd/nerdctl/image_encrypt_linux_test.go

@@ -17,12 +17,16 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"testing"
 
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
 	"github.com/containerd/nerdctl/pkg/buildkitutil"
 	"github.com/containerd/nerdctl/pkg/testutil"
 	"github.com/containerd/nerdctl/pkg/testutil/testregistry"
@@ -63,11 +67,46 @@ func newJWEKeyPair(t testing.TB) *jweKeyPair {
 }
 
 func rmiAll(base *testutil.Base) {
+	base.T.Logf("Pruning images")
 	imageIDs := base.Cmd("images", "--no-trunc", "-a", "-q").OutLines()
 	base.Cmd(append([]string{"rmi", "-f"}, imageIDs...)...).AssertOK()
+
+	base.T.Logf("Pruning build caches")
 	if _, err := buildkitutil.GetBuildkitHost(testutil.Namespace); err == nil {
 		base.Cmd("builder", "prune").AssertOK()
 	}
+
+	// For BuildKit >= 0.11, pruning cache isn't enough to remove manifest blobs that are referred by build history blobs
+	// https://github.com/containerd/nerdctl/pull/1833
+	if base.Target == testutil.Nerdctl {
+		base.T.Logf("Pruning all content blobs")
+		addr := base.ContainerdAddress()
+		client, err := containerd.New(addr, containerd.WithDefaultNamespace(testutil.Namespace))
+		assert.NilError(base.T, err)
+		cs := client.ContentStore()
+		ctx := context.TODO()
+		wf := func(info content.Info) error {
+			base.T.Logf("Pruning blob %+v", info)
+			if err := cs.Delete(ctx, info.Digest); err != nil {
+				base.T.Log(err)
+			}
+			return nil
+		}
+		if err := cs.Walk(ctx, wf); err != nil {
+			base.T.Log(err)
+		}
+
+		base.T.Logf("Pruning all images (again?)")
+		is := client.ImageService()
+		imgs, err := is.List(ctx)
+		assert.NilError(base.T, err)
+		for _, img := range imgs {
+			base.T.Logf("Pruning image %+v", img)
+			if err := is.Delete(ctx, img.Name, images.SynchronousDelete()); err != nil {
+				base.T.Log(err)
+			}
+		}
+	}
 }
 
 func TestImageEncryptJWE(t *testing.T) {

cmd/nerdctl/system_prune.go

@@ -18,13 +18,16 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
+	"os/exec"
 	"strings"
 
 	"github.com/containerd/nerdctl/pkg/api/types"
+	"github.com/containerd/nerdctl/pkg/buildkitutil"
 	"github.com/containerd/nerdctl/pkg/clientutil"
 	"github.com/containerd/nerdctl/pkg/cmd/volume"
-	buildkitclient "github.com/moby/buildkit/client"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -131,47 +134,45 @@ func systemPruneAction(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-type cacheUsageInfo struct {
-	ID   string
-	Size int64
-}
-
-func buildCachePrune(ctx context.Context, cmd *cobra.Command, pruneAll bool, namespace string) ([]cacheUsageInfo, error) {
+func buildCachePrune(ctx context.Context, cmd *cobra.Command, pruneAll bool, namespace string) ([]buildkitutil.UsageInfo, error) {
 	buildkitHost, err := getBuildkitHost(cmd, namespace)
 	if err != nil {
 		return nil, err
 	}
-	opts := []buildkitclient.ClientOpt{buildkitclient.WithFailFast()}
-	client, err := buildkitclient.New(ctx, buildkitHost, opts)
+	buildctlBinary, err := buildkitutil.BuildctlBinary()
 	if err != nil {
 		return nil, err
 	}
-	var pruneOpts []buildkitclient.PruneOption
+	buildctlArgs := buildkitutil.BuildctlBaseArgs(buildkitHost)
+	buildctlArgs = append(buildctlArgs, "prune", "--format={{json .}}")
 	if pruneAll {
-		pruneOpts = append(pruneOpts, buildkitclient.PruneAll)
+		buildctlArgs = append(buildctlArgs, "--all")
 	}
-
-	var usageInfos []buildkitclient.UsageInfo
-	usageCh := make(chan buildkitclient.UsageInfo)
-
-	go func() {
-		for item := range usageCh {
-			usageInfos = append(usageInfos, item)
-		}
-	}()
-
-	err = client.Prune(ctx, usageCh, pruneOpts...)
-	close(usageCh)
+	buildctlCmd := exec.Command(buildctlBinary, buildctlArgs...)
+	logrus.Debugf("running %v", buildctlCmd.Args)
+	buildctlCmd.Stderr = cmd.ErrOrStderr()
+	stdout, err := buildctlCmd.StdoutPipe()
 	if err != nil {
-		return nil, err
-	}
-
-	result := make([]cacheUsageInfo, len(usageInfos))
-	for i, item := range usageInfos {
-		result[i] = cacheUsageInfo{
-			ID:   item.ID,
-			Size: item.Size,
+		return nil, fmt.Errorf("faild to get stdout piper for %v: %w", buildctlCmd.Args, err)
+	}
+	defer stdout.Close()
+	if err = buildctlCmd.Start(); err != nil {
+		return nil, fmt.Errorf("faild to start %v: %w", buildctlCmd.Args, err)
+	}
+	dec := json.NewDecoder(stdout)
+	result := make([]buildkitutil.UsageInfo, 0)
+	for {
+		var v buildkitutil.UsageInfo
+		if err := dec.Decode(&v); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, fmt.Errorf("faild to decode output from %v: %w", buildctlCmd.Args, err)
 		}
+		result = append(result, v)
 	}
+	if err = buildctlCmd.Wait(); err != nil {
+		return nil, fmt.Errorf("faild to wait for %v to complete: %w", buildctlCmd.Args, err)
+	}
+
 	return result, nil
 }

go.mod

@@ -33,7 +33,6 @@ require (
 	github.com/ipfs/go-cid v0.3.2
 	github.com/mattn/go-isatty v0.0.17
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/moby/buildkit v0.10.6
 	github.com/moby/sys/mount v0.3.3
 	github.com/moby/sys/signal v0.7.0
 	github.com/opencontainers/go-digest v1.0.0
@@ -61,7 +60,6 @@ require (
 
 require (
 	github.com/containerd/cgroups/v3 v3.0.0-20221112182753-e8802a182774 // indirect
-	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
 	google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect
 )
 
@@ -76,23 +74,17 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/distribution/distribution/v3 v3.0.0-20221103125252-ebfa2a0ac0a9 // indirect
 	github.com/djherbis/times v1.5.0 // indirect
-	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/frankban/quicktest v1.14.2 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
-	github.com/gofrs/flock v0.8.1 // indirect
-	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -122,20 +114,15 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tinylib/msgp v1.1.6 // indirect
-	github.com/tonistiigi/fsutil v0.0.0-20220315205639-9ed612626da3 // indirect
 	github.com/urfave/cli v1.22.10 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 // indirect
 	go.opentelemetry.io/otel v1.11.1 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 // indirect
-	go.opentelemetry.io/otel/sdk v1.11.1 // indirect
 	go.opentelemetry.io/otel/trace v1.11.1 // indirect
-	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	golang.org/x/mod v0.6.0 // indirect
 	golang.org/x/tools v0.2.0 // indirect
 	google.golang.org/grpc v1.51.0 // indirect