Add a new manifest command to support multi-architecture builds (#1705

)

Implement Pack Manifest commands in support of https://github.com/buildpacks/rfcs/blob/main/text/0124-pack-manifest-list-commands.md

Signed-off-by: Husni Faiz <ahamedhusni73@gmail.com>
Signed-off-by: WYGIN <wygininc@gmail.com>
Signed-off-by: sai kiran <wyginc1@gmail.com>
Signed-off-by: Juan Bustamante <jbustamante@vmware.com>
Signed-off-by: Juan Bustamante <juan.bustamante@broadcom.com>
Signed-off-by: Sai Kiran Maggidi <107541780+WYGIN@users.noreply.github.com>
Signed-off-by: Juan Bustamante <bustamantejj@gmail.com>

Co-authored-by: Juan Bustamante <jbustamante@vmware.com>
Co-authored-by: WYGIN <wygininc@gmail.com>
Co-authored-by: sai kiran <wyginc1@gmail.com>
Co-authored-by: Juan Bustamante <juan.bustamante@broadcom.com>
Co-authored-by: Sai Kiran Maggidi <107541780+WYGIN@users.noreply.github.com>
Co-authored-by: Juan Bustamante <bustamantejj@gmail.com>

acceptance/acceptance_test.go

@@ -18,10 +18,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/buildpacks/imgutil"
 	"github.com/buildpacks/lifecycle/api"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
 	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 	"github.com/pelletier/go-toml"
 	"github.com/sclevine/spec"
 	"github.com/sclevine/spec/report"
@@ -623,6 +626,147 @@ func testWithoutSpecificBuilderRequirement(
 			})
 		})
 	})
+
+	when("manifest", func() {
+		var (
+			indexRepoName  string
+			repoName1      string
+			repoName2      string
+			indexLocalPath string
+			tmpDir         string
+			err            error
+		)
+
+		it.Before(func() {
+			h.SkipIf(t, !pack.SupportsFeature(invoke.ManifestCommands), "pack manifest commands are available since 0.34.0")
+
+			// local storage path
+			tmpDir, err = os.MkdirTemp("", "manifest-commands-test")
+			assert.Nil(err)
+			os.Setenv("XDG_RUNTIME_DIR", tmpDir)
+
+			// manifest commands are experimental
+			pack.EnableExperimental()
+
+			// used to avoid authentication issues with the local registry
+			os.Setenv("DOCKER_CONFIG", registryConfig.DockerConfigDir)
+		})
+
+		it.After(func() {
+			assert.Succeeds(os.RemoveAll(tmpDir))
+		})
+
+		when("create", func() {
+			it.Before(func() {
+				it.Before(func() {
+					indexRepoName = registryConfig.RepoName(h.NewRandomIndexRepoName())
+
+					// Manifest 1
+					repoName1 = fmt.Sprintf("%s:%s", indexRepoName, "busybox-amd64")
+					h.CreateRemoteImage(t, indexRepoName, "busybox-amd64", "busybox@sha256:a236a6469768c17ca1a6ac81a35fe6fbc1efd76b0dcdf5aebb1cf5f0774ee539")
+
+					// Manifest 2
+					repoName2 = fmt.Sprintf("%s:%s", indexRepoName, "busybox-arm64")
+					h.CreateRemoteImage(t, indexRepoName, "busybox-arm64", "busybox@sha256:0bcc1b827b855c65eaf6e031e894e682b6170160b8a676e1df7527a19d51fb1a")
+				})
+			})
+			when("--publish", func() {
+				it("creates and push the index to a remote registry", func() {
+					output := pack.RunSuccessfully("manifest", "create", "--publish", indexRepoName, repoName1, repoName2)
+					assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(indexRepoName)
+					h.AssertRemoteImageIndex(t, indexRepoName, types.OCIImageIndex, 2)
+				})
+			})
+
+			when("no --publish", func() {
+				it("creates the index locally", func() {
+					output := pack.RunSuccessfully("manifest", "create", indexRepoName, repoName1, repoName2)
+					assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexLocallyCreated(indexRepoName)
+
+					indexLocalPath = filepath.Join(tmpDir, imgutil.MakeFileSafeName(indexRepoName))
+					index := h.ReadIndexManifest(t, indexLocalPath)
+					h.AssertEq(t, len(index.Manifests), 2)
+					h.AssertEq(t, index.MediaType, types.OCIImageIndex)
+				})
+			})
+		})
+
+		when("index is already created", func() {
+			var digest v1.Hash
+
+			it.Before(func() {
+				indexRepoName = registryConfig.RepoName(h.NewRandomIndexRepoName())
+
+				// Manifest 1
+				repoName1 = fmt.Sprintf("%s:%s", indexRepoName, "busybox-amd64")
+				image1 := h.CreateRemoteImage(t, indexRepoName, "busybox-amd64", "busybox@sha256:a236a6469768c17ca1a6ac81a35fe6fbc1efd76b0dcdf5aebb1cf5f0774ee539")
+				digest, err = image1.Digest()
+				assert.Nil(err)
+
+				// Manifest 2
+				repoName2 = fmt.Sprintf("%s:%s", indexRepoName, "busybox-arm64")
+				h.CreateRemoteImage(t, indexRepoName, "busybox-arm64", "busybox@sha256:0bcc1b827b855c65eaf6e031e894e682b6170160b8a676e1df7527a19d51fb1a")
+
+				// create an index locally
+				pack.RunSuccessfully("manifest", "create", indexRepoName, repoName1)
+				indexLocalPath = filepath.Join(tmpDir, imgutil.MakeFileSafeName(indexRepoName))
+			})
+
+			when("add", func() {
+				it("adds the manifest to the index", func() {
+					output := pack.RunSuccessfully("manifest", "add", indexRepoName, repoName2)
+					assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulManifestAddedToIndex(repoName2)
+
+					index := h.ReadIndexManifest(t, indexLocalPath)
+					h.AssertEq(t, len(index.Manifests), 2)
+					h.AssertEq(t, index.MediaType, types.OCIImageIndex)
+				})
+			})
+
+			when("remove", func() {
+				it("removes the index from local storage", func() {
+					output := pack.RunSuccessfully("manifest", "remove", indexRepoName)
+					assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexDeleted()
+
+					h.AssertPathDoesNotExists(t, indexLocalPath)
+				})
+			})
+
+			when("annotate", func() {
+				it("adds annotations to the manifest in the index", func() {
+					output := pack.RunSuccessfully("manifest", "annotate", indexRepoName, repoName1, "--annotations", "foo=bar")
+					assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexAnnotated(repoName1, indexRepoName)
+
+					index := h.ReadIndexManifest(t, indexLocalPath)
+					h.AssertEq(t, len(index.Manifests), 1)
+					h.AssertEq(t, len(index.Manifests[0].Annotations), 1)
+				})
+			})
+
+			when("rm", func() {
+				it.Before(func() {
+					// we need to point to the manifest digest we want to delete
+					repoName1 = fmt.Sprintf("%s@%s", repoName1, digest.String())
+				})
+
+				it("removes the manifest from the index", func() {
+					output := pack.RunSuccessfully("manifest", "rm", indexRepoName, repoName1)
+					assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulRemoveManifestFromIndex(indexRepoName)
+
+					index := h.ReadIndexManifest(t, indexLocalPath)
+					h.AssertEq(t, len(index.Manifests), 0)
+				})
+			})
+
+			when("push", func() {
+				it("pushes the index to a remote registry", func() {
+					output := pack.RunSuccessfully("manifest", "push", indexRepoName)
+					assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(indexRepoName)
+					h.AssertRemoteImageIndex(t, indexRepoName, types.OCIImageIndex, 1)
+				})
+			})
+		})
+	})
 }
 
 func testAcceptance(

acceptance/assertions/output.go

@@ -32,6 +32,42 @@ func (o OutputAssertionManager) ReportsSuccessfulImageBuild(name string) {
 	o.assert.ContainsF(o.output, "Successfully built image '%s'", name)
 }
 
+func (o OutputAssertionManager) ReportsSuccessfulIndexLocallyCreated(name string) {
+	o.testObject.Helper()
+
+	o.assert.ContainsF(o.output, "Successfully created manifest list '%s'", name)
+}
+
+func (o OutputAssertionManager) ReportsSuccessfulIndexPushed(name string) {
+	o.testObject.Helper()
+
+	o.assert.ContainsF(o.output, "Successfully pushed manifest list '%s' to registry", name)
+}
+
+func (o OutputAssertionManager) ReportsSuccessfulManifestAddedToIndex(name string) {
+	o.testObject.Helper()
+
+	o.assert.ContainsF(o.output, "Successfully added image '%s' to index", name)
+}
+
+func (o OutputAssertionManager) ReportsSuccessfulIndexDeleted() {
+	o.testObject.Helper()
+
+	o.assert.Contains(o.output, "Successfully deleted manifest list(s) from local storage")
+}
+
+func (o OutputAssertionManager) ReportsSuccessfulIndexAnnotated(name, manifest string) {
+	o.testObject.Helper()
+
+	o.assert.ContainsF(o.output, "Successfully annotated image '%s' in index '%s'", name, manifest)
+}
+
+func (o OutputAssertionManager) ReportsSuccessfulRemoveManifestFromIndex(name string) {
+	o.testObject.Helper()
+
+	o.assert.ContainsF(o.output, "Successfully removed image(s) from index: '%s'", name)
+}
+
 func (o OutputAssertionManager) ReportSuccessfulQuietBuild(name string) {
 	o.testObject.Helper()
 	o.testObject.Log("quiet mode")

acceptance/invoke/pack.go

@@ -238,6 +238,7 @@ const (
 	PlatformRetries
 	FlattenBuilderCreationV2
 	FixesRunImageMetadata
+	ManifestCommands
 )
 
 var featureTests = map[Feature]func(i *PackInvoker) bool{
@@ -274,6 +275,9 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
 	FixesRunImageMetadata: func(i *PackInvoker) bool {
 		return i.atLeast("v0.34.0")
 	},
+	ManifestCommands: func(i *PackInvoker) bool {
+		return i.atLeast("v0.34.0")
+	},
 }
 
 func (i *PackInvoker) SupportsFeature(f Feature) bool {

cmd/cmd.go

@@ -104,6 +104,7 @@ func NewPackCommand(logger ConfigurableLogger) (*cobra.Command, error) {
 		rootCmd.AddCommand(commands.SetDefaultRegistry(logger, cfg, cfgPath))
 		rootCmd.AddCommand(commands.RemoveRegistry(logger, cfg, cfgPath))
 		rootCmd.AddCommand(commands.YankBuildpack(logger, cfg, packClient))
+		rootCmd.AddCommand(commands.NewManifestCommand(logger, packClient))
 	}
 
 	packHome, err := config.PackHome()

go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/apex/log v1.9.0
-	github.com/buildpacks/imgutil v0.0.0-20240422175901-30b002586ecc
+	github.com/buildpacks/imgutil v0.0.0-20240507132533-9f7b96c3d09d
 	github.com/buildpacks/lifecycle v0.19.4
 	github.com/docker/cli v26.0.1+incompatible
 	github.com/docker/docker v26.1.1+incompatible

go.sum

@@ -91,8 +91,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/buildpacks/imgutil v0.0.0-20240422175901-30b002586ecc h1:Pxgpm4bIjgWY9a8r8xHlMYp6wzm5hIZHtw4Iux3XA0c=
-github.com/buildpacks/imgutil v0.0.0-20240422175901-30b002586ecc/go.mod h1:n2R6VRuWsAX3cyHCp/u0Z4WJcixny0gYg075J39owrk=
+github.com/buildpacks/imgutil v0.0.0-20240507132533-9f7b96c3d09d h1:GVRuY/C8j4pjOddeeZelbKKLMMX+dYR3TlxE4L1hECU=
+github.com/buildpacks/imgutil v0.0.0-20240507132533-9f7b96c3d09d/go.mod h1:n2R6VRuWsAX3cyHCp/u0Z4WJcixny0gYg075J39owrk=
 github.com/buildpacks/lifecycle v0.19.4 h1:lATTWyMs4m4+3+n6PgvmB0lBgut2+b8eZ+iu/wXwhrI=
 github.com/buildpacks/lifecycle v0.19.4/go.mod h1:sWrBJzf/7dWrcHrWiV/P2+3jS8G8Ki5tczq8jO3XVRQ=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=

internal/commands/buildpack_inspect.go

@@ -42,7 +42,7 @@ func BuildpackInspect(logger logging.Logger, cfg config.Config, client PackClien
 	return cmd
 }
 
-func buildpackInspect(logger logging.Logger, buildpackName, registryName string, flags BuildpackInspectFlags, cfg config.Config, pack PackClient) error {
+func buildpackInspect(logger logging.Logger, buildpackName, registryName string, flags BuildpackInspectFlags, _ config.Config, pack PackClient) error {
 	logger.Infof("Inspecting buildpack: %s\n", style.Symbol(buildpackName))
 
 	inspectedBuildpacksOutput, err := inspectAllBuildpacks(

internal/commands/commands.go

@@ -7,6 +7,7 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/google/go-containerregistry/pkg/v1/types"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
@@ -32,6 +33,13 @@ type PackClient interface {
 	InspectExtension(client.InspectExtensionOptions) (*client.ExtensionInfo, error)
 	PullBuildpack(context.Context, client.PullBuildpackOptions) error
 	DownloadSBOM(name string, options client.DownloadSBOMOptions) error
+	CreateManifest(ctx context.Context, opts client.CreateManifestOptions) error
+	AnnotateManifest(ctx context.Context, opts client.ManifestAnnotateOptions) error
+	AddManifest(ctx context.Context, opts client.ManifestAddOptions) error
+	DeleteManifest(name []string) error
+	RemoveManifest(name string, images []string) error
+	PushManifest(client.PushManifestOptions) error
+	InspectManifest(string) error
 }
 
 func AddHelpFlag(cmd *cobra.Command, commandName string) {
@@ -107,3 +115,16 @@ func isTrustedBuilder(cfg config.Config, builder string) bool {
 func deprecationWarning(logger logging.Logger, oldCmd, replacementCmd string) {
 	logger.Warnf("Command %s has been deprecated, please use %s instead", style.Symbol("pack "+oldCmd), style.Symbol("pack "+replacementCmd))
 }
+
+func parseFormatFlag(value string) (types.MediaType, error) {
+	var format types.MediaType
+	switch value {
+	case "oci":
+		format = types.OCIImageIndex
+	case "docker":
+		format = types.DockerManifestList
+	default:
+		return format, errors.Errorf("%s invalid media type format", value)
+	}
+	return format, nil
+}

internal/commands/extension_inspect.go

@@ -26,7 +26,7 @@ func ExtensionInspect(logger logging.Logger, cfg config.Config, client PackClien
 	return cmd
 }
 
-func extensionInspect(logger logging.Logger, extensionName string, cfg config.Config, pack PackClient) error {
+func extensionInspect(logger logging.Logger, extensionName string, _ config.Config, pack PackClient) error {
 	logger.Infof("Inspecting extension: %s\n", style.Symbol(extensionName))
 
 	inspectedExtensionsOutput, err := inspectAllExtensions(

internal/commands/manifest.go

@@ -0,0 +1,33 @@
+package commands
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/buildpacks/pack/pkg/logging"
+)
+
+func NewManifestCommand(logger logging.Logger, client PackClient) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "manifest",
+		Short: "Interact with OCI image indexes",
+		Long: `An image index is a higher-level manifest which points to specific image manifests and is ideal for one or more platforms; see: https://github.com/opencontainers/image-spec/ for more details
+
+'pack manifest' commands provide tooling to create, update, or delete images indexes or push them to a remote registry.
+'pack' will save a local copy of the image index at '$PACK_HOME/manifests'; the environment variable 'XDG_RUNTIME_DIR' 
+can be set to override the location, allowing manifests to be edited locally before being pushed to a registry.
+
+These commands are experimental. For more information, consult the RFC which can be found at https://github.com/buildpacks/rfcs/blob/main/text/0124-pack-manifest-list-commands.md`,
+		RunE: nil,
+	}
+
+	cmd.AddCommand(ManifestCreate(logger, client))
+	cmd.AddCommand(ManifestAdd(logger, client))
+	cmd.AddCommand(ManifestAnnotate(logger, client))
+	cmd.AddCommand(ManifestDelete(logger, client))
+	cmd.AddCommand(ManifestInspect(logger, client))
+	cmd.AddCommand(ManifestPush(logger, client))
+	cmd.AddCommand(ManifestRemove(logger, client))
+
+	AddHelpFlag(cmd, "manifest")
+	return cmd
+}

internal/commands/manifest_add.go

@@ -0,0 +1,27 @@
+package commands
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/buildpacks/pack/pkg/client"
+	"github.com/buildpacks/pack/pkg/logging"
+)
+
+// ManifestAdd adds a new image to a manifest list (image index).
+func ManifestAdd(logger logging.Logger, pack PackClient) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "add [OPTIONS] <manifest-list> <manifest> [flags]",
+		Args:    cobra.MatchAll(cobra.ExactArgs(2), cobra.OnlyValidArgs),
+		Short:   "Add an image to a manifest list.",
+		Example: `pack manifest add my-image-index my-image:some-arch`,
+		RunE: logError(logger, func(cmd *cobra.Command, args []string) (err error) {
+			return pack.AddManifest(cmd.Context(), client.ManifestAddOptions{
+				IndexRepoName: args[0],
+				RepoName:      args[1],
+			})
+		}),
+	}
+
+	AddHelpFlag(cmd, "add")
+	return cmd
+}