From 081e875d8caf273fdcee2208131b8a2cf97aa27f Mon Sep 17 00:00:00 2001 From: Jacob Blain Christen Date: Fri, 13 Sep 2024 01:58:27 -0700 Subject: [PATCH] enable docker load for hauler tarballs - fixes #276 Signed-off-by: Jacob Blain Christen --- cmd/hauler/cli/store/add.go | 145 +++++++++++++++++++++++++++++++++++- pkg/consts/consts.go | 5 +- pkg/content/oci.go | 4 +- pkg/store/store.go | 2 +- 4 files changed, 150 insertions(+), 6 deletions(-) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index dab32b8..ae168cd 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -1,15 +1,26 @@ package store import ( + "bytes" "context" + "encoding/json" + "path" + "slices" + referencev3 "github.com/distribution/distribution/v3/reference" "github.com/google/go-containerregistry/pkg/name" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" + libv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + imagev1 "github.com/opencontainers/image-spec/specs-go/v1" "helm.sh/helm/v3/pkg/action" "hauler.dev/go/hauler/internal/flags" "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1alpha1" "hauler.dev/go/hauler/pkg/artifacts/file" + "hauler.dev/go/hauler/pkg/artifacts/file/getter" + "hauler.dev/go/hauler/pkg/consts" "hauler.dev/go/hauler/pkg/content/chart" "hauler.dev/go/hauler/pkg/cosign" "hauler.dev/go/hauler/pkg/log" @@ -86,6 +97,138 @@ func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform return nil } + l.Debugf("adding 'image' [%s] to the export manifest", r.Name()) + + var ( + xm = make(tarball.Manifest, 0) + xr = make(map[string]*tarball.Descriptor) + ) + + exportImage := func(refname string, desc libv1.Descriptor, index libv1.ImageIndex) error { + digest := desc.Digest.String() + image, err := index.Image(desc.Digest) + if err != nil { + return err + } + + config, err := image.ConfigName() + if err != nil { + return err + } + + xd, recorded := xr[digest] + if !recorded { + xm = append(xm, tarball.Descriptor{ + Config: path.Join(imagev1.ImageBlobsDir, config.Algorithm, config.Hex), + RepoTags: []string{}, + Layers: []string{}, + }) + xd = &xm[len(xm)-1] + + layers, err := image.Layers() + if err != nil { + return err + } + for _, layer := range layers { + ldg, err := layer.Digest() + if err != nil { + return err + } + xd.Layers = append(xd.Layers[:], path.Join(imagev1.ImageBlobsDir, ldg.Algorithm, ldg.Hex)) + } + } + + ref, err := name.ParseReference(refname) + if err != nil { + return err + } + + switch tag := ref.(type) { + case name.Tag: + named, err := referencev3.ParseNormalizedNamed(refname) + if err != nil { + return err + } + named = referencev3.TagNameOnly(named) + repotag := referencev3.FamiliarString(named) + xd.RepoTags = append(xd.RepoTags[:], repotag) + slices.Sort(xd.RepoTags) + xd.RepoTags = slices.Compact(xd.RepoTags) + ref = tag.Digest(digest) + } + xr[digest] = xd + l.Debugf("image [%s]: type=%s, size=%d", ref.Name(), desc.MediaType, desc.Size) + + return nil + } + + root, err := layout.FromPath(s.Root) + if err != nil { + return err + } + + riix, err := root.ImageIndex() + if err != nil { + return err + } + + rixm, err := riix.IndexManifest() + if err != nil { + return err + } + + for _, desc := range rixm.Manifests { + l.Debugf("descriptor [%s] >>> %s", desc.Digest.String(), desc.MediaType) + if artifactType := types.MediaType(desc.ArtifactType); artifactType != "" && !artifactType.IsImage() && !artifactType.IsIndex() { + l.Debugf("descriptor [%s] <<< SKIPPING ARTIFACT (%q)", desc.Digest.String(), desc.ArtifactType) + continue + } + if desc.Annotations != nil { + // we only care about images that cosign has added to the layout index + if kind, hasKind := desc.Annotations[consts.KindAnnotationName]; hasKind { + if refName, hasRefName := desc.Annotations[imagev1.AnnotationRefName]; hasRefName { + // branch on image (aka image manifest) or image index + switch kind { + case consts.KindAnnotationImage: + if err := exportImage(refName, desc, riix); err != nil { + return err + } + case consts.KindAnnotationIndex: + l.Debugf("index [%s]: digest=%s, type=%s, size=%d", refName, desc.Digest.String(), desc.MediaType, desc.Size) + iix, err := riix.ImageIndex(desc.Digest) + if err != nil { + return err + } + ixm, err := iix.IndexManifest() + if err != nil { + return err + } + for _, ixd := range ixm.Manifests { + if ixd.MediaType.IsImage() { + if err := exportImage(refName, ixd, iix); err != nil { + return err + } + } + } + default: + l.Debugf("descriptor [%s] <<< SKIPPING KIND (%q)", desc.Digest.String(), kind) + } + } + } + } + } + + buf := bytes.Buffer{} + err = json.NewEncoder(&buf).Encode(xm) + if err != nil { + return err + } + + err = root.WriteFile("manifest.json", buf.Bytes(), 0666) + if err != nil { + return err + } + l.Infof("successfully added 'image' [%s]", r.Name()) return nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 579ed4e..9e4f23b 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -47,8 +47,9 @@ const ( HaulerVendorPrefix = "vnd.hauler" OCIImageIndexFile = "index.json" - KindAnnotationName = "kind" - KindAnnotation = "dev.cosignproject.cosign/image" + KindAnnotationName = "kind" + KindAnnotationImage = "dev.cosignproject.cosign/image" + KindAnnotationIndex = "dev.cosignproject.cosign/imageIndex" CarbideRegistry = "rgcrprod.azurecr.us" ImageAnnotationKey = "hauler.dev/key" diff --git a/pkg/content/oci.go b/pkg/content/oci.go index 99ae586..092d2d5 100644 --- a/pkg/content/oci.go +++ b/pkg/content/oci.go @@ -124,9 +124,9 @@ func (o *OCI) SaveIndex() error { kindJ := descs[j].Annotations["kind"] // Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top. - if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) { + if strings.HasPrefix(kindI, consts.KindAnnotationImage) && !strings.HasPrefix(kindJ, consts.KindAnnotationImage) { return true - } else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) { + } else if !strings.HasPrefix(kindI, consts.KindAnnotationImage) && strings.HasPrefix(kindJ, consts.KindAnnotationImage) { return false } return false // Default: maintain the order. diff --git a/pkg/store/store.go b/pkg/store/store.go index d1b5dcc..1e7ea91 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -118,7 +118,7 @@ func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (oci Digest: digest.FromBytes(mdata), Size: int64(len(mdata)), Annotations: map[string]string{ - consts.KindAnnotationName: consts.KindAnnotation, + consts.KindAnnotationName: consts.KindAnnotationImage, ocispec.AnnotationRefName: ref, }, URLs: nil,