Skip to content

Commit

Permalink
enable docker load for hauler tarballs
Browse files Browse the repository at this point in the history
- fixes #276

Signed-off-by: Jacob Blain Christen <jacob.blain.christen@ranchergovernment.com>
  • Loading branch information
dweomer committed Sep 16, 2024
1 parent 5aa55e9 commit f1d8955
Show file tree
Hide file tree
Showing 4 changed files with 173 additions and 5 deletions.
167 changes: 167 additions & 0 deletions cmd/hauler/cli/store/save.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,25 @@
package store

import (
"bytes"
"context"
"encoding/json"
"os"
"path"
"path/filepath"
"slices"

referencev3 "github.com/distribution/distribution/v3/reference"
"github.com/google/go-containerregistry/pkg/name"
libv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/mholt/archiver/v3"
imagev1 "github.com/opencontainers/image-spec/specs-go/v1"

"hauler.dev/go/hauler/internal/flags"
"hauler.dev/go/hauler/pkg/consts"
"hauler.dev/go/hauler/pkg/log"
)

Expand All @@ -34,6 +46,10 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, outputFile string) error {
return err
}

if err := writeExportsManifest(ctx, "."); err != nil {
return err
}

err = a.Archive([]string{"."}, absOutputfile)
if err != nil {
return err
Expand All @@ -42,3 +58,154 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, outputFile string) error {
l.Infof("saved store [%s] -> [%s]", o.StoreDir, absOutputfile)
return nil
}

type exports struct {
digests []string
records map[string]tarball.Descriptor
}

func (x *exports) describe() tarball.Manifest {
m := make(tarball.Manifest, len(x.digests))
for i, d := range x.digests {
m[i] = x.records[d]
}
return m
}

func writeExportsManifest(ctx context.Context, dir string) error {
l := log.FromContext(ctx)

oci, err := layout.FromPath(dir)
if err != nil {
return err
}

idx, err := oci.ImageIndex()
if err != nil {
return err
}

imx, err := idx.IndexManifest()
if err != nil {
return err
}

x := &exports{
digests: []string{},
records: map[string]tarball.Descriptor{},
}

for _, desc := range imx.Manifests {
l.Debugf("descriptor [%s] >>> %s", desc.Digest.String(), desc.MediaType)
if artifactType := types.MediaType(desc.ArtifactType); artifactType != "" && !artifactType.IsImage() && !artifactType.IsIndex() {
l.Debugf("descriptor [%s] <<< SKIPPING ARTIFACT (%q)", desc.Digest.String(), desc.ArtifactType)
continue
}
if desc.Annotations != nil {
// we only care about images that cosign has added to the layout index
if kind, hasKind := desc.Annotations[consts.KindAnnotationName]; hasKind {
if refName, hasRefName := desc.Annotations[imagev1.AnnotationRefName]; hasRefName {
// branch on image (aka image manifest) or image index
switch kind {
case consts.KindAnnotationImage:
if err := x.record(ctx, idx, desc, refName); err != nil {
return err
}
case consts.KindAnnotationIndex:
l.Debugf("index [%s]: digest=%s, type=%s, size=%d", refName, desc.Digest.String(), desc.MediaType, desc.Size)
iix, err := idx.ImageIndex(desc.Digest)
if err != nil {
return err
}
ixm, err := iix.IndexManifest()
if err != nil {
return err
}
for _, ixd := range ixm.Manifests {
if ixd.MediaType.IsImage() {
if err := x.record(ctx, iix, ixd, refName); err != nil {
return err
}
}
}
default:
l.Debugf("descriptor [%s] <<< SKIPPING KIND (%q)", desc.Digest.String(), kind)
}
}
}
}
}

buf := bytes.Buffer{}
mnf := x.describe()
err = json.NewEncoder(&buf).Encode(mnf)
if err != nil {
return err
}

return oci.WriteFile("manifest.json", buf.Bytes(), 0666)
}

func (x *exports) record(ctx context.Context, index libv1.ImageIndex, desc libv1.Descriptor, refname string) error {
l := log.FromContext(ctx)

digest := desc.Digest.String()
image, err := index.Image(desc.Digest)
if err != nil {
return err
}

config, err := image.ConfigName()
if err != nil {
return err
}

xd, recorded := x.records[digest]
if !recorded {
// record one export record per digest
x.digests = append(x.digests, digest)
xd = tarball.Descriptor{
Config: path.Join(imagev1.ImageBlobsDir, config.Algorithm, config.Hex),
RepoTags: []string{},
Layers: []string{},
}

layers, err := image.Layers()
if err != nil {
return err
}
for _, layer := range layers {
xl, err := layer.Digest()
if err != nil {
return err
}
xd.Layers = append(xd.Layers[:], path.Join(imagev1.ImageBlobsDir, xl.Algorithm, xl.Hex))
}
}

ref, err := name.ParseReference(refname)
if err != nil {
return err
}

// record tags for the digest, eliminating dupes
switch tag := ref.(type) {
case name.Tag:
named, err := referencev3.ParseNormalizedNamed(refname)
if err != nil {
return err
}
named = referencev3.TagNameOnly(named)
repotag := referencev3.FamiliarString(named)
xd.RepoTags = append(xd.RepoTags[:], repotag)
slices.Sort(xd.RepoTags)
xd.RepoTags = slices.Compact(xd.RepoTags)
ref = tag.Digest(digest)
}

l.Debugf("image [%s]: type=%s, size=%d", ref.Name(), desc.MediaType, desc.Size)
// record export descriptor for the digest
x.records[digest] = xd

return nil
}
5 changes: 3 additions & 2 deletions pkg/consts/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@ const (
HaulerVendorPrefix = "vnd.hauler"
OCIImageIndexFile = "index.json"

KindAnnotationName = "kind"
KindAnnotation = "dev.cosignproject.cosign/image"
KindAnnotationName = "kind"
KindAnnotationImage = "dev.cosignproject.cosign/image"
KindAnnotationIndex = "dev.cosignproject.cosign/imageIndex"

CarbideRegistry = "rgcrprod.azurecr.us"
ImageAnnotationKey = "hauler.dev/key"
Expand Down
4 changes: 2 additions & 2 deletions pkg/content/oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,9 @@ func (o *OCI) SaveIndex() error {
kindJ := descs[j].Annotations["kind"]

// Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top.
if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) {
if strings.HasPrefix(kindI, consts.KindAnnotationImage) && !strings.HasPrefix(kindJ, consts.KindAnnotationImage) {
return true
} else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) {
} else if !strings.HasPrefix(kindI, consts.KindAnnotationImage) && strings.HasPrefix(kindJ, consts.KindAnnotationImage) {
return false
}
return false // Default: maintain the order.
Expand Down
2 changes: 1 addition & 1 deletion pkg/store/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (oci
Digest: digest.FromBytes(mdata),
Size: int64(len(mdata)),
Annotations: map[string]string{
consts.KindAnnotationName: consts.KindAnnotation,
consts.KindAnnotationName: consts.KindAnnotationImage,
ocispec.AnnotationRefName: ref,
},
URLs: nil,
Expand Down

0 comments on commit f1d8955

Please sign in to comment.