New flatten buildpacks/builder implementation - Part 2 - Implementing RFC-0123

builder/buildpack_identifier.go

@@ -0,0 +1,5 @@
+package builder
+
+type Identifier interface {
+	Id() string
+}

internal/builder/builder.go

@@ -68,24 +68,23 @@ const (
 
 // Builder represents a pack builder, used to build images
 type Builder struct {
-	baseImageName            string
-	buildConfigEnv           map[string]string
-	image                    imgutil.Image
-	layerWriterFactory       archive.TarWriterFactory
-	lifecycle                Lifecycle
-	lifecycleDescriptor      LifecycleDescriptor
-	additionalBuildpacks     buildpack.ManagedCollection
-	additionalExtensions     buildpack.ManagedCollection
-	flattenExcludeBuildpacks []string
-	metadata                 Metadata
-	mixins                   []string
-	env                      map[string]string
-	uid, gid                 int
-	StackID                  string
-	replaceOrder             bool
-	order                    dist.Order
-	orderExtensions          dist.Order
-	validateMixins           bool
+	baseImageName        string
+	buildConfigEnv       map[string]string
+	image                imgutil.Image
+	layerWriterFactory   archive.TarWriterFactory
+	lifecycle            Lifecycle
+	lifecycleDescriptor  LifecycleDescriptor
+	additionalBuildpacks buildpack.ManagedCollection
+	additionalExtensions buildpack.ManagedCollection
+	metadata             Metadata
+	mixins               []string
+	env                  map[string]string
+	uid, gid             int
+	StackID              string
+	replaceOrder         bool
+	order                dist.Order
+	orderExtensions      dist.Order
+	validateMixins       bool
 }
 
 type orderTOML struct {
@@ -103,8 +102,7 @@ type moduleWithDiffID struct {
 type BuilderOption func(*options) error
 
 type options struct {
-	flatten bool
-	exclude []string
+	modules buildpack.FlattenModuleInfos
 }
 
 // FromImage constructs a builder from a builder image
@@ -142,17 +140,16 @@ func constructBuilder(img imgutil.Image, newName string, errOnMissingLabel bool,
 	}
 
 	bldr := &Builder{
-		baseImageName:            img.Name(),
-		image:                    img,
-		layerWriterFactory:       layerWriterFactory,
-		metadata:                 metadata,
-		lifecycleDescriptor:      constructLifecycleDescriptor(metadata),
-		env:                      map[string]string{},
-		buildConfigEnv:           map[string]string{},
-		validateMixins:           true,
-		additionalBuildpacks:     *buildpack.NewModuleManager(opts.flatten),
-		additionalExtensions:     *buildpack.NewModuleManager(opts.flatten),
-		flattenExcludeBuildpacks: opts.exclude,
+		baseImageName:        img.Name(),
+		image:                img,
+		layerWriterFactory:   layerWriterFactory,
+		metadata:             metadata,
+		lifecycleDescriptor:  constructLifecycleDescriptor(metadata),
+		env:                  map[string]string{},
+		buildConfigEnv:       map[string]string{},
+		validateMixins:       true,
+		additionalBuildpacks: buildpack.NewModuleManagerV2(opts.modules),
+		additionalExtensions: buildpack.NewModuleManagerV2(opts.modules),
 	}
 
 	if err := addImgLabelsToBuildr(bldr); err != nil {
@@ -166,17 +163,9 @@ func constructBuilder(img imgutil.Image, newName string, errOnMissingLabel bool,
 	return bldr, nil
 }
 
-func WithFlatten() BuilderOption {
+func WithFlatten(modules buildpack.FlattenModuleInfos) BuilderOption {
 	return func(o *options) error {
-		o.flatten = true
-		return nil
-	}
-}
-
-func WithFlattenExclude(exclude []string) BuilderOption {
-	return func(o *options) error {
-		o.flatten = true
-		o.exclude = exclude
+		o.modules = modules
 		return nil
 	}
 }
@@ -306,14 +295,14 @@ func (b *Builder) AllModules(kind string) []buildpack.BuildModule {
 	return b.moduleManager(kind).AllModules()
 }
 
-func (b *Builder) moduleManager(kind string) *buildpack.ManagedCollection {
+func (b *Builder) moduleManager(kind string) buildpack.ManagedCollection {
 	switch kind {
 	case buildpack.KindBuildpack:
-		return &b.additionalBuildpacks
+		return b.additionalBuildpacks
 	case buildpack.KindExtension:
-		return &b.additionalExtensions
+		return b.additionalExtensions
 	}
-	return &buildpack.ManagedCollection{}
+	return buildpack.NewModuleManager(false)
 }
 
 func (b *Builder) FlattenedModules(kind string) [][]buildpack.BuildModule {
@@ -662,15 +651,14 @@ func (b *Builder) addFlattenedModules(kind string, logger logging.Logger, tmpDir
 	)
 
 	buildModuleWriter := buildpack.NewBuildModuleWriter(logger, b.layerWriterFactory)
-	excludedModules := buildpack.Set(b.flattenExcludeBuildpacks)
 
 	for i, additionalModules := range flattenModules {
 		modFlattenTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s-flatten", kind, strconv.Itoa(i)))
 		if err := os.MkdirAll(modFlattenTmpDir, os.ModePerm); err != nil {
 			return nil, errors.Wrap(err, "creating flatten temp dir")
 		}
 
-		finalTarPath, buildModuleExcluded, err = buildModuleWriter.NToLayerTar(modFlattenTmpDir, fmt.Sprintf("%s-flatten-%s", kind, strconv.Itoa(i)), additionalModules, excludedModules)
+		finalTarPath, buildModuleExcluded, err = buildModuleWriter.NToLayerTar(modFlattenTmpDir, fmt.Sprintf("%s-flatten-%s", kind, strconv.Itoa(i)), additionalModules, nil)
 		if err != nil {
 			return nil, errors.Wrapf(err, "writing layer %s", finalTarPath)
 		}

internal/builder/builder_test.go

@@ -1874,7 +1874,10 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
 		when("all", func() {
 			it.Before(func() {
 				var err error
-				bldr, err = builder.New(builderImage, "some-builder", builder.WithFlatten())
+				flattenModules, err := buildpack.ParseFlattenBuildModules([]string{"buildpack-1-id@buildpack-1-version-1,buildpack-1-id@buildpack-1-version-2,buildpack-2-id@buildpack-2-version-1"})
+				h.AssertNil(t, err)
+
+				bldr, err = builder.New(builderImage, "some-builder", builder.WithFlatten(flattenModules))
 				h.AssertNil(t, err)
 
 				// Let's add some buildpacks

internal/commands/builder_create.go

@@ -3,27 +3,26 @@ package commands
 import (
 	"fmt"
 	"path/filepath"
-	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
 	"github.com/buildpacks/pack/builder"
 	"github.com/buildpacks/pack/internal/config"
 	"github.com/buildpacks/pack/internal/style"
+	"github.com/buildpacks/pack/pkg/buildpack"
 	"github.com/buildpacks/pack/pkg/client"
 	"github.com/buildpacks/pack/pkg/image"
 	"github.com/buildpacks/pack/pkg/logging"
 )
 
 // BuilderCreateFlags define flags provided to the CreateBuilder command
 type BuilderCreateFlags struct {
-	Flatten         bool
 	Publish         bool
 	BuilderTomlPath string
 	Registry        string
 	Policy          string
-	FlattenExclude  []string
+	Flatten         []string
 }
 
 // CreateBuilder creates a builder image, based on a builder config
@@ -82,6 +81,11 @@ Creating a custom builder allows you to control what buildpacks are used and wha
 				return err
 			}
 
+			flattenBuildpacks, err := buildpack.ParseFlattenBuildModules(flags.Flatten)
+			if err != nil {
+				return err
+			}
+
 			imageName := args[0]
 			if err := pack.CreateBuilder(cmd.Context(), client.CreateBuilderOptions{
 				RelativeBaseDir: relativeBaseDir,
@@ -91,8 +95,7 @@ Creating a custom builder allows you to control what buildpacks are used and wha
 				Publish:         flags.Publish,
 				Registry:        flags.Registry,
 				PullPolicy:      pullPolicy,
-				Flatten:         flags.Flatten,
-				FlattenExclude:  flags.FlattenExclude,
+				Flatten:         flattenBuildpacks,
 			}); err != nil {
 				return err
 			}
@@ -109,8 +112,7 @@ Creating a custom builder allows you to control what buildpacks are used and wha
 	cmd.Flags().StringVarP(&flags.BuilderTomlPath, "config", "c", "", "Path to builder TOML file (required)")
 	cmd.Flags().BoolVar(&flags.Publish, "publish", false, "Publish to registry")
 	cmd.Flags().StringVar(&flags.Policy, "pull-policy", "", "Pull policy to use. Accepted values are always, never, and if-not-present. The default is always")
-	cmd.Flags().BoolVar(&flags.Flatten, "flatten", false, "Flatten each composite buildpack into a single layer")
-	cmd.Flags().StringSliceVarP(&flags.FlattenExclude, "flatten-exclude", "e", nil, "Buildpacks to exclude from flattening, in the form of '<buildpack-id>@<buildpack-version>'")
+	cmd.Flags().StringSliceVar(&flags.Flatten, "flatten", nil, "List of Buildpacks to flatten together in one layer (format: '<buildpack-id>@<buildpack-version>,<buildpack-id>@<buildpack-version>'")
 
 	AddHelpFlag(cmd, "create")
 	return cmd
@@ -133,13 +135,5 @@ func validateCreateFlags(flags *BuilderCreateFlags, cfg config.Config) error {
 		return errors.Errorf("Please provide a builder config path, using --config.")
 	}
 
-	if flags.Flatten && len(flags.FlattenExclude) > 0 {
-		for _, exclude := range flags.FlattenExclude {
-			if strings.Count(exclude, "@") != 1 {
-				return errors.Errorf("invalid format %s; please use '<buildpack-id>@<buildpack-version>' to exclude buildpack from flattening", exclude)
-			}
-		}
-	}
-
 	return nil
 }

pkg/buildpack/build_module_info.go

@@ -0,0 +1,65 @@
+package buildpack
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/buildpacks/pack/pkg/dist"
+)
+
+type ModuleInfos interface {
+	BuildModule() []dist.ModuleInfo
+}
+
+type FlattenModuleInfos interface {
+	FlattenModules() []ModuleInfos
+}
+
+type flattenModules struct {
+	modules []ModuleInfos
+}
+
+func (fl *flattenModules) FlattenModules() []ModuleInfos {
+	return fl.modules
+}
+
+type buildModuleInfosImpl struct {
+	modules []dist.ModuleInfo
+}
+
+func (b *buildModuleInfosImpl) BuildModule() []dist.ModuleInfo {
+	return b.modules
+}
+
+func ParseFlattenBuildModules(buildpacksID []string) (FlattenModuleInfos, error) {
+	var buildModuleInfos []ModuleInfos
+	for _, ids := range buildpacksID {
+		modules, err := parseBuildpackName(ids)
+		if err != nil {
+			return nil, err
+		}
+		buildModuleInfos = append(buildModuleInfos, modules)
+	}
+	return &flattenModules{modules: buildModuleInfos}, nil
+}
+
+func parseBuildpackName(names string) (ModuleInfos, error) {
+	var buildModuleInfos []dist.ModuleInfo
+	ids := strings.Split(names, ",")
+	for _, id := range ids {
+		if strings.Count(id, "@") != 1 {
+			return nil, errors.Errorf("invalid format %s; please use '<buildpack-id>@<buildpack-version>' to add buildpacks to be flatten", id)
+		}
+		bpFullName := strings.Split(id, "@")
+		if len(bpFullName) != 2 {
+			return nil, errors.Errorf("invalid format %s; '<buildpack-id>' and '<buildpack-version>' must be specified", id)
+		}
+		bpID := dist.ModuleInfo{
+			ID:      bpFullName[0],
+			Version: bpFullName[1],
+		}
+		buildModuleInfos = append(buildModuleInfos, bpID)
+	}
+	return &buildModuleInfosImpl{modules: buildModuleInfos}, nil
+}

pkg/buildpack/builder.go

@@ -102,7 +102,7 @@ func NewBuilder(imageFactory ImageFactory, ops ...PackageBuilderOption) *Package
 	moduleManager := NewModuleManager(opts.flatten)
 	return &PackageBuilder{
 		imageFactory:             imageFactory,
-		dependencies:             *moduleManager,
+		dependencies:             moduleManager,
 		flattenAllBuildpacks:     opts.flatten,
 		flattenExcludeBuildpacks: opts.exclude,
 		logger:                   opts.logger,