Flatten feature implementation

acceptance/acceptance_test.go

@@ -2308,92 +2308,6 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 						})
 					})
 				})
-
-				when("build --buildpack <flattened buildpack>", func() {
-					var (
-						tmpDir                         string
-						flattenedPackageName           string
-						simplePackageConfigFixtureName = "package.toml"
-					)
-
-					generateAggregatePackageToml := func(buildpackURI, nestedPackageName, operatingSystem string) string {
-						t.Helper()
-						packageTomlFile, err := os.CreateTemp(tmpDir, "package_aggregate-*.toml")
-						assert.Nil(err)
-
-						pack.FixtureManager().TemplateFixtureToFile(
-							"package_aggregate.toml",
-							packageTomlFile,
-							map[string]interface{}{
-								"BuildpackURI": buildpackURI,
-								"PackageName":  nestedPackageName,
-								"OS":           operatingSystem,
-							},
-						)
-
-						assert.Nil(packageTomlFile.Close())
-						return packageTomlFile.Name()
-					}
-
-					it.Before(func() {
-						h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
-						h.SkipIf(t, imageManager.HostOS() == "windows", "buildpack directories not supported on windows")
-
-						var err error
-						tmpDir, err = os.MkdirTemp("", "buildpack-package-flattened-tests")
-						assert.Nil(err)
-
-						buildpackManager = buildpacks.NewBuildModuleManager(t, assert)
-						buildpackManager.PrepareBuildModules(tmpDir, buildpacks.BpSimpleLayersParent, buildpacks.BpSimpleLayers)
-
-						// set up a flattened buildpack
-						packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
-						nestedPackageName := "test/flattened-package-" + h.RandString(10)
-						nestedPackage := buildpacks.NewPackageImage(
-							t,
-							pack,
-							nestedPackageName,
-							packageTomlPath,
-							buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
-						)
-						buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
-						assertImage.ExistsLocally(nestedPackageName)
-
-						aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
-						flattenedPackageName = "test/package-" + h.RandString(10)
-
-						_ = pack.RunSuccessfully(
-							"buildpack", "package", flattenedPackageName,
-							"-c", aggregatePackageToml,
-							"--flatten",
-						)
-
-						assertImage.ExistsLocally(flattenedPackageName)
-						assertImage.HasLengthLayers(flattenedPackageName, 1)
-					})
-
-					it.After(func() {
-						assert.Nil(os.RemoveAll(tmpDir))
-						imageManager.CleanupImages(flattenedPackageName)
-					})
-
-					when("--flatten", func() {
-						it("does not write duplicate tar files when creating the ephemeral builder", func() {
-							output := pack.RunSuccessfully(
-								"build", repoName,
-								"-p", filepath.Join("testdata", "mock_app"),
-								"--buildpack", fmt.Sprintf("docker://%s", flattenedPackageName),
-								"--builder", builderName,
-							)
-							// buildpack returning an empty tar file is non-deterministic,
-							// but we expect one of them to throw the warning
-							h.AssertContainsMatch(t, output, "Buildpack '(simple/layers@simple-layers-version|simple/layers/parent@simple-layers-parent-version)' is a component of a flattened buildpack that will be added elsewhere, skipping...")
-
-							// simple/layers BP exists on the builder and in the flattened buildpack
-							h.AssertContainsMatch(t, output, "Buildpack 'simple/layers@simple-layers-version' already exists on builder with same contents, skipping...")
-						})
-					})
-				})
 			})
 
 			when("inspecting builder", func() {
@@ -2890,6 +2804,17 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 				})
 			})
 		})
+
+		when("create", func() {
+			when("--flatten=<buildpacks>", func() {
+				it.Focus("should flatten together all buildpacks specified", func() {
+					output, err := createFlattenBuilder(t, assert, buildpackManager, lifecycle, createBuilderPack)
+					h.AssertNil(t, err)
+					assertImage.ExistsLocally(output)
+					assertImage.HasLengthLayers(output, 6)
+				})
+			})
+		})
 	})
 }
 
@@ -3045,6 +2970,166 @@ func createComplexBuilder(t *testing.T,
 	return bldr, nil
 }
 
+func createFlattenBuilder(
+	t *testing.T,
+	assert h.AssertionManager,
+	buildpackManager buildpacks.BuildModuleManager,
+	lifecycle config.LifecycleAsset,
+	pack *invoke.PackInvoker,
+) (string, error) {
+	t.Helper()
+
+	t.Log("creating flatten builder...")
+
+	tmpRootDir := createTempRootDir(assert, "create-test-flatten-builder")
+	defer os.RemoveAll(tmpRootDir)
+
+	packageBuildpack1 := createBuildpackLayer(
+		t,
+		assert,
+		tmpRootDir,
+		pack,
+		createBuildpack("simple-layers-package-image-buildpack1"),
+		"simple-layers-1",
+		"simple-layers-flatten-buildpack1.tgz",
+		buildpacks.BpSimpleLayersFlatten1,
+	)
+	packageBuildpack2 := createBuildpackLayer(
+		t,
+		assert,
+		tmpRootDir,
+		pack,
+		createBuildpack("simple-layers-package-image-buildpack2"),
+		"simple-layers-2",
+		"simple-layers-flatten-buildpack2.tgz",
+		buildpacks.BpSimpleLayersFlatten2,
+	)
+	packageBuildpack3 := createBuildpackLayer(
+		t,
+		assert,
+		tmpRootDir,
+		pack,
+		createBuildpack("simple-layers-package-image-buildpack3"),
+		"simple-layers-3",
+		"simple-layers-flatten-buildpack3.tgz",
+		buildpacks.BpSimpleLayersFlatten3,
+	)
+	packageBuildpack4 := createBuildpackLayer(
+		t,
+		assert,
+		tmpRootDir,
+		pack,
+		createBuildpack("simple-layers-package-image-buildpack4"),
+		"simple-layers-4",
+		"simple-layers-flatten-buildpack4.tgz",
+		buildpacks.BpSimpleLayersFlatten4,
+	)
+
+	buildpackManager.PrepareBuildModules(tmpRootDir, packageBuildpack1, packageBuildpack2, packageBuildpack3, packageBuildpack4)
+
+	builderConfigFileName := createFlattenBuilderToml(t, assert, lifecycle, tmpRootDir, pack)
+
+	builderName := registryConfig.RepoName("test/flatten-builder-" + h.RandString(10))
+	output := pack.RunSuccessfully(
+		"builder", "create", builderName,
+		"-c", builderConfigFileName,
+		"--no-color",
+		"--flatten=simple-layers-package-image-buildpack1:simple-layers-flatten-version,simple-layers-package-image-buildpack2:simple-layers-flatten-version",
+	)
+
+	assert.Contains(output, fmt.Sprintf("Successfully created builder image '%s'", builderName))
+	assert.Succeeds(h.PushImage(dockerCli, builderName, registryConfig))
+
+	return builderName, nil
+}
+
+func createFlattenBuilderToml(
+	t *testing.T,
+	assert h.AssertionManager,
+	lifecycle config.LifecycleAsset,
+	tmpRootDir string,
+	pack *invoke.PackInvoker,
+) string {
+	var templateMapping = make(map[string]interface{})
+	if lifecycle.HasLocation() {
+		lifecycleURI := lifecycle.EscapedPath()
+		t.Logf("adding lifecycle path '%s' to flatten builder config", lifecycleURI)
+		templateMapping["lifecycle_uri"] = lifecycleURI
+	} else {
+		lifecycleVersion := lifecycle.Version()
+		t.Logf("adding lifecycle version '%s' to flatten builder config", lifecycleVersion)
+		templateMapping["lifecycle_version"] = lifecycleVersion
+	}
+
+	builderConfigFile, err := os.CreateTemp(tmpRootDir, "builder_flatten.toml")
+	assert.Nil(err)
+
+	pack.FixtureManager().TemplateFixtureToFile("builder_flatten.toml", builderConfigFile, templateMapping)
+
+	err = builderConfigFile.Close()
+	assert.Nil(err)
+
+	return builderConfigFile.Name()
+}
+
+func createBuildpackLayer(
+	t *testing.T,
+	assert h.AssertionManager,
+	tmpRootDirPath string,
+	pack *invoke.PackInvoker,
+	packageImageName string,
+	tempWorkingDir string,
+	packageURI string,
+	archiveBuildModule buildpacks.TestBuildModule,
+) buildpacks.PackageImage {
+	layerDir := createTempWorkingDirectory(filepath.Join(tmpRootDirPath, tempWorkingDir), assert)
+	layerPackageToml := createTempPackageTomlFile(layerDir, assert)
+	fillFlattenPackageToml(layerPackageToml, pack, packageURI)
+
+	return buildpacks.NewPackageImage(
+		t,
+		pack,
+		packageImageName,
+		layerPackageToml.Name(),
+		buildpacks.WithRequiredBuildpacks(
+			archiveBuildModule,
+		),
+	)
+}
+
+func createTempRootDir(assert h.AssertionManager, name string) string {
+	tmpRootDir, err := os.MkdirTemp("", name)
+	assert.Nil(err)
+	return tmpRootDir
+}
+
+func fillFlattenPackageToml(layersPackageToml *os.File, pack *invoke.PackInvoker, uri string) {
+	pack.FixtureManager().TemplateFixtureToFile(
+		"package-flatten.toml",
+		layersPackageToml,
+		map[string]interface{}{
+			"OS":  imageManager.HostOS(),
+			"URI": uri,
+		},
+	)
+}
+
+func createTempPackageTomlFile(tmpDir string, assert h.AssertionManager) *os.File {
+	packageTomlFile, err := os.CreateTemp(tmpDir, "package-*.toml")
+	assert.Nil(err)
+	return packageTomlFile
+}
+
+func createBuildpack(name string) string {
+	return registryConfig.RepoName(name + "-" + h.RandString(8))
+}
+
+func createTempWorkingDirectory(name string, assert h.AssertionManager) string {
+	err := os.Mkdir(name, os.ModePerm)
+	assert.Nil(err)
+	return name
+}
+
 func createBuilder(
 	t *testing.T,
 	assert h.AssertionManager,

acceptance/buildpacks/archive_buildpack.go

@@ -87,6 +87,10 @@ func (a archiveBuildModule) createTgz(sourceDir string) (string, error) {
 var (
 	BpSimpleLayersParent       = &archiveBuildModule{name: "simple-layers-parent-buildpack"}
 	BpSimpleLayers             = &archiveBuildModule{name: "simple-layers-buildpack"}
+	BpSimpleLayersFlatten1     = &archiveBuildModule{name: "simple-layers-flatten-buildpack1"}
+	BpSimpleLayersFlatten2     = &archiveBuildModule{name: "simple-layers-flatten-buildpack2"}
+	BpSimpleLayersFlatten3     = &archiveBuildModule{name: "simple-layers-flatten-buildpack3"}
+	BpSimpleLayersFlatten4     = &archiveBuildModule{name: "simple-layers-flatten-buildpack4"}
 	BpSimpleLayersDifferentSha = &archiveBuildModule{name: "simple-layers-buildpack-different-sha"}
 	BpInternetCapable          = &archiveBuildModule{name: "internet-capable-buildpack"}
 	BpReadVolume               = &archiveBuildModule{name: "read-volume-buildpack"}

acceptance/invoke/pack.go

@@ -233,7 +233,6 @@ const (
 	RunImageExtensions
 	StackValidation
 	ForceRebase
-	BuildpackFlatten
 	MetaBuildpackFolder
 )
 
@@ -256,9 +255,6 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
 	ForceRebase: func(i *PackInvoker) bool {
 		return i.atLeast("v0.30.0")
 	},
-	BuildpackFlatten: func(i *PackInvoker) bool {
-		return i.atLeast("v0.30.0")
-	},
 	MetaBuildpackFolder: func(i *PackInvoker) bool {
 		return i.atLeast("v0.30.0")
 	},

acceptance/testdata/mock_buildpacks/simple-layers-flatten-buildpack1/bin/build

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+echo "---> Build: Simple Layers Buildpack"
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+launch_dir=$1
+
+## makes a launch layer
+echo "making launch layer"
+
+# Add color line, to test for --no-color
+echo "Color: Styled"
+
+mkdir "$launch_dir/launch-layer"
+echo "Launch Dep Contents" > "$launch_dir/launch-layer/launch-dep"
+ln -snf "$launch_dir/launch-layer" launch-deps
+echo "[types]" > "$launch_dir/launch-layer.toml"
+echo "launch = true" >> "$launch_dir/launch-layer.toml"
+
+## makes a cached launch layer
+if [[ ! -f "$launch_dir/cached-launch-layer.toml" ]]; then
+    echo "making cached launch layer"
+    mkdir "$launch_dir/cached-launch-layer"
+    echo "Cached Dep Contents" > "$launch_dir/cached-launch-layer/cached-dep"
+    ln -snf "$launch_dir/cached-launch-layer" cached-deps
+    echo "[types]" > "$launch_dir/cached-launch-layer.toml"
+    echo "launch = true" >> "$launch_dir/cached-launch-layer.toml"
+    echo "cache = true" >> "$launch_dir/cached-launch-layer.toml"
+else
+  echo "reusing cached launch layer"
+  echo "[types]" > "$launch_dir/cached-launch-layer.toml"
+  echo "launch = true" >> "$launch_dir/cached-launch-layer.toml"
+  echo "cache = true" >> "$launch_dir/cached-launch-layer.toml"
+  ln -snf "$launch_dir/cached-launch-layer" cached-deps
+fi
+
+## adds a process
+cat <<EOF > "$launch_dir/launch.toml"
+[[processes]]
+  type = "web"
+  command = "./run"
+  args = ["8080"]
+  default = true
+
+[[processes]]
+  type = "hello"
+  command = "echo"
+  args = ["hello", "world"]
+  direct = true
+EOF
+
+echo "---> Done"