Merge pull request #4134 from jedevc/enable-multi-exporters

Enable multiple exporters (alternative)

Author: jedevc

api/services/control/control.pb.go

@@ -60,8 +60,11 @@ message UsageRecord {
 message SolveRequest {
 	string Ref = 1;
 	pb.Definition Definition = 2;
-	string Exporter = 3;
-	map<string, string> ExporterAttrs = 4;
+	// ExporterDeprecated and ExporterAttrsDeprecated are deprecated in favor
+	// of the new Exporters. If these fields are set, then they will be
+	// appended to the Exporters field if Exporters was not explicitly set.
+	string ExporterDeprecated = 3;
+	map<string, string> ExporterAttrsDeprecated = 4;
 	string Session = 5;
 	string Frontend = 6;
 	map<string, string> FrontendAttrs = 7;
@@ -70,6 +73,7 @@ message SolveRequest {
 	map<string, pb.Definition> FrontendInputs = 10;
 	bool Internal = 11; // Internal builds are not recorded in build history
 	moby.buildkit.v1.sourcepolicy.Policy SourcePolicy = 12;
+	repeated Exporter Exporters = 13;
 }
 
 message CacheOptions {
@@ -227,11 +231,15 @@ message Descriptor {
 }
 
 message BuildResultInfo {
-	Descriptor Result = 1;
+	Descriptor ResultDeprecated = 1;
 	repeated Descriptor Attestations = 2;
+	map<int64, Descriptor> Results = 3;
 }
 
+// Exporter describes the output exporter
 message Exporter {
+	// Type identifies the exporter
 	string Type = 1;
+	// Attrs specifies exporter configuration
 	map<string, string> Attrs = 2;
 }

api/services/control/control.proto

@@ -46,6 +46,7 @@ import (
 	gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/session/secrets/secretsprovider"
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/errdefs"
@@ -152,6 +153,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testTarExporterWithSocketCopy,
 	testTarExporterSymlink,
 	testMultipleRegistryCacheImportExport,
+	testMultipleExporters,
 	testSourceMap,
 	testSourceMapFromRef,
 	testLazyImagePush,
@@ -2569,6 +2571,106 @@ func testUser(t *testing.T, sb integration.Sandbox) {
 	checkAllReleasable(t, c, sb, true)
 }
 
+func testMultipleExporters(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	def, err := llb.Scratch().File(llb.Mkfile("foo.txt", 0o755, nil)).Marshal(context.TODO())
+	require.NoError(t, err)
+
+	destDir, destDir2 := t.TempDir(), t.TempDir()
+	out := filepath.Join(destDir, "out.tar")
+	outW, err := os.Create(out)
+	require.NoError(t, err)
+	defer outW.Close()
+
+	out2 := filepath.Join(destDir, "out2.tar")
+	outW2, err := os.Create(out2)
+	require.NoError(t, err)
+	defer outW2.Close()
+
+	registry, err := sb.NewRegistry()
+	if errors.Is(err, integration.ErrRequirements) {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+
+	target1, target2 := registry+"/buildkit/build/exporter:image",
+		registry+"/buildkit/build/alternative:image"
+
+	imageExporter := ExporterImage
+	if workers.IsTestDockerd() {
+		imageExporter = "moby"
+	}
+
+	ref := identity.NewID()
+	resp, err := c.Solve(sb.Context(), def, SolveOpt{
+		Ref: ref,
+		Exports: []ExportEntry{
+			{
+				Type: imageExporter,
+				Attrs: map[string]string{
+					"name": target1,
+				},
+			},
+			{
+				Type: imageExporter,
+				Attrs: map[string]string{
+					"name":           target2,
+					"oci-mediatypes": "true",
+				},
+			},
+			// Ensure that multiple local exporter destinations are written properly
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir2,
+			},
+			// Ensure that multiple instances of the same exporter are possible
+			{
+				Type:   ExporterTar,
+				Output: fixedWriteCloser(outW),
+			},
+			{
+				Type:   ExporterTar,
+				Output: fixedWriteCloser(outW2),
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, resp.ExporterResponse["image.name"], target2)
+	require.FileExists(t, filepath.Join(destDir, "out.tar"))
+	require.FileExists(t, filepath.Join(destDir, "out2.tar"))
+	require.FileExists(t, filepath.Join(destDir, "foo.txt"))
+	require.FileExists(t, filepath.Join(destDir2, "foo.txt"))
+
+	history, err := c.ControlClient().ListenBuildHistory(sb.Context(), &controlapi.BuildHistoryRequest{
+		Ref:       ref,
+		EarlyExit: true,
+	})
+	require.NoError(t, err)
+	for {
+		ev, err := history.Recv()
+		if err != nil {
+			require.Equal(t, io.EOF, err)
+			break
+		}
+		require.Equal(t, ref, ev.Record.Ref)
+
+		require.Len(t, ev.Record.Result.Results, 2)
+		require.Equal(t, images.MediaTypeDockerSchema2Manifest, ev.Record.Result.Results[0].MediaType)
+		require.Equal(t, ocispecs.MediaTypeImageManifest, ev.Record.Result.Results[1].MediaType)
+		require.Equal(t, ev.Record.Result.Results[0], ev.Record.Result.ResultDeprecated)
+	}
+}
+
 func testOCIExporter(t *testing.T, sb integration.Sandbox) {
 	workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter)
 	requiresLinux(t)
@@ -6979,7 +7081,7 @@ func testMergeOpCache(t *testing.T, sb integration.Sandbox, mode string) {
 
 	for i, layer := range manifest.Layers {
 		_, err = contentStore.Info(ctx, layer.Digest)
-		require.ErrorIs(t, err, ctderrdefs.ErrNotFound, "unexpected error %v for index %d", err, i)
+		require.ErrorIs(t, err, ctderrdefs.ErrNotFound, "unexpected error %v for index %d (%s)", err, i, layer.Digest)
 	}
 
 	// re-run the build with a change only to input1 using the remote cache
@@ -9659,7 +9761,7 @@ var hostNetwork integration.ConfigUpdater = &netModeHost{}
 var defaultNetwork integration.ConfigUpdater = &netModeDefault{}
 var bridgeDNSNetwork integration.ConfigUpdater = &netModeBridgeDNS{}
 
-func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
+func fixedWriteCloser(wc io.WriteCloser) filesync.FileOutputFunc {
 	return func(map[string]string) (io.WriteCloser, error) {
 		return wc, nil
 	}

client/client_test.go

@@ -56,8 +56,8 @@ type SolveOpt struct {
 type ExportEntry struct {
 	Type      string
 	Attrs     map[string]string
-	Output    func(map[string]string) (io.WriteCloser, error) // for ExporterOCI and ExporterDocker
-	OutputDir string                                          // for ExporterLocal
+	Output    filesync.FileOutputFunc // for ExporterOCI and ExporterDocker
+	OutputDir string                  // for ExporterLocal
 }
 
 type CacheOptionsEntry struct {
@@ -130,14 +130,6 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		return nil, err
 	}
 
-	var ex ExportEntry
-	if len(opt.Exports) > 1 {
-		return nil, errors.New("currently only single Exports can be specified")
-	}
-	if len(opt.Exports) == 1 {
-		ex = opt.Exports[0]
-	}
-
 	storesToUpdate := []string{}
 
 	if !opt.SessionPreInitialized {
@@ -161,58 +153,63 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			contentStores[key2] = store
 		}
 
-		var supportFile bool
-		var supportDir bool
-		switch ex.Type {
-		case ExporterLocal:
-			supportDir = true
-		case ExporterTar:
-			supportFile = true
-		case ExporterOCI, ExporterDocker:
-			supportDir = ex.OutputDir != ""
-			supportFile = ex.Output != nil
-		}
-
-		if supportFile && supportDir {
-			return nil, errors.Errorf("both file and directory output is not supported by %s exporter", ex.Type)
-		}
-		if !supportFile && ex.Output != nil {
-			return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
-		}
-		if !supportDir && ex.OutputDir != "" {
-			return nil, errors.Errorf("output directory is not supported by %s exporter", ex.Type)
-		}
-
-		if supportFile {
-			if ex.Output == nil {
-				return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type)
-			}
-			s.Allow(filesync.NewFSSyncTarget(ex.Output))
-		}
-		if supportDir {
-			if ex.OutputDir == "" {
-				return nil, errors.Errorf("output directory is required for %s exporter", ex.Type)
-			}
+		var syncTargets []filesync.FSSyncTarget
+		for exID, ex := range opt.Exports {
+			var supportFile bool
+			var supportDir bool
 			switch ex.Type {
+			case ExporterLocal:
+				supportDir = true
+			case ExporterTar:
+				supportFile = true
 			case ExporterOCI, ExporterDocker:
-				if err := os.MkdirAll(ex.OutputDir, 0755); err != nil {
-					return nil, err
+				supportDir = ex.OutputDir != ""
+				supportFile = ex.Output != nil
+			}
+			if supportFile && supportDir {
+				return nil, errors.Errorf("both file and directory output is not supported by %s exporter", ex.Type)
+			}
+			if !supportFile && ex.Output != nil {
+				return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
+			}
+			if !supportDir && ex.OutputDir != "" {
+				return nil, errors.Errorf("output directory is not supported by %s exporter", ex.Type)
+			}
+			if supportFile {
+				if ex.Output == nil {
+					return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type)
 				}
-				cs, err := contentlocal.NewStore(ex.OutputDir)
-				if err != nil {
-					return nil, err
+				syncTargets = append(syncTargets, filesync.WithFSSync(exID, ex.Output))
+			}
+			if supportDir {
+				if ex.OutputDir == "" {
+					return nil, errors.Errorf("output directory is required for %s exporter", ex.Type)
+				}
+				switch ex.Type {
+				case ExporterOCI, ExporterDocker:
+					if err := os.MkdirAll(ex.OutputDir, 0755); err != nil {
+						return nil, err
+					}
+					cs, err := contentlocal.NewStore(ex.OutputDir)
+					if err != nil {
+						return nil, err
+					}
+					contentStores["export"] = cs
+					storesToUpdate = append(storesToUpdate, ex.OutputDir)
+				default:
+					syncTargets = append(syncTargets, filesync.WithFSSyncDir(exID, ex.OutputDir))
 				}
-				contentStores["export"] = cs
-				storesToUpdate = append(storesToUpdate, ex.OutputDir)
-			default:
-				s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
 			}
 		}
 
 		if len(contentStores) > 0 {
 			s.Allow(sessioncontent.NewAttachable(contentStores))
 		}
 
+		if len(syncTargets) > 0 {
+			s.Allow(filesync.NewFSSyncTarget(syncTargets...))
+		}
+
 		eg.Go(func() error {
 			sd := c.sessionDialer
 			if sd == nil {
@@ -260,19 +257,34 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			frontendInputs[key] = def.ToPB()
 		}
 
+		exports := make([]*controlapi.Exporter, 0, len(opt.Exports))
+		exportDeprecated := ""
+		exportAttrDeprecated := map[string]string{}
+		for i, exp := range opt.Exports {
+			if i == 0 {
+				exportDeprecated = exp.Type
+				exportAttrDeprecated = exp.Attrs
+			}
+			exports = append(exports, &controlapi.Exporter{
+				Type:  exp.Type,
+				Attrs: exp.Attrs,
+			})
+		}
+
 		resp, err := c.ControlClient().Solve(ctx, &controlapi.SolveRequest{
-			Ref:            ref,
-			Definition:     pbd,
-			Exporter:       ex.Type,
-			ExporterAttrs:  ex.Attrs,
-			Session:        s.ID(),
-			Frontend:       opt.Frontend,
-			FrontendAttrs:  frontendAttrs,
-			FrontendInputs: frontendInputs,
-			Cache:          cacheOpt.options,
-			Entitlements:   opt.AllowedEntitlements,
-			Internal:       opt.Internal,
-			SourcePolicy:   opt.SourcePolicy,
+			Ref:                     ref,
+			Definition:              pbd,
+			Exporters:               exports,
+			ExporterDeprecated:      exportDeprecated,
+			ExporterAttrsDeprecated: exportAttrDeprecated,
+			Session:                 s.ID(),
+			Frontend:                opt.Frontend,
+			FrontendAttrs:           frontendAttrs,
+			FrontendInputs:          frontendInputs,
+			Cache:                   cacheOpt.options,
+			Entitlements:            opt.AllowedEntitlements,
+			Internal:                opt.Internal,
+			SourcePolicy:            opt.SourcePolicy,
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to solve")

client/solve.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/containerd/console"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/session/filesync"
 	"github.com/pkg/errors"
 )
 
@@ -66,7 +67,7 @@ func ParseOutput(exports []string) ([]client.ExportEntry, error) {
 }
 
 // resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path).
-func resolveExporterDest(exporter, dest string, attrs map[string]string) (func(map[string]string) (io.WriteCloser, error), string, error) {
+func resolveExporterDest(exporter, dest string, attrs map[string]string) (filesync.FileOutputFunc, string, error) {
 	wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
 		return func(m map[string]string) (io.WriteCloser, error) {
 			return wc, nil