diff --git a/oci/artefact.go b/oci/artefact.go index 061c0cb9..8a2f2b7d 100644 --- a/oci/artefact.go +++ b/oci/artefact.go @@ -1,23 +1,46 @@ package oci import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "encoding/hex" "fmt" "io" + "maps" "os" "path/filepath" "time" ociclient "github.com/fluxcd/pkg/oci" + "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/google/go-containerregistry/pkg/compression" "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" typesv1 "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/docker/labs-brown-tape/manifest/types" + attestTypes "github.com/docker/labs-brown-tape/attest/types" + manifestTypes "github.com/docker/labs-brown-tape/manifest/types" +) + +const ( + mediaTypePrefix = "application/vnd.docker.tape" + ConfigMediaType = mediaTypePrefix + ".config.v1alpha1+json" + ContentMediaType = mediaTypePrefix + ".content.v1alpha1.tar+gzip" + AttestMediaType = mediaTypePrefix + ".attest.v1alpha1.jsonl+gzip" + + ContentInterpreterAnnotation = mediaTypePrefix + "content-interpreter.v1alpha1" + ContentInterpreterKubectlApply = mediaTypePrefix + ".kubectl-apply.v1alpha1.tar+gzip" + + // TODO: content inteprete invocation with an image + + regularFileMode = 0o640 ) type ArtefactInfo struct { @@ -64,7 +87,7 @@ func (c *Client) GetArtefact(ctx context.Context, ref string) (*ArtefactInfo, er } // based on https://github.com/fluxcd/pkg/blob/2a323d771e17af02dee2ccbbb9b445b78ab048e5/oci/client/push.go -func (c *Client) PushArtefact(ctx context.Context, destinationRef, sourceDir string, timestamp *time.Time) (string, error) { +func (c *Client) PushArtefact(ctx context.Context, destinationRef, sourceDir string, timestamp *time.Time, sourceAttestations ...attestTypes.Statement) (string, error) { tmpDir, err := os.MkdirTemp("", "bpt-oci-artefact-*") if err != nil { return "", err @@ -73,60 +96,98 @@ func (c *Client) PushArtefact(ctx context.Context, destinationRef, sourceDir str tmpFile := filepath.Join(tmpDir, "artefact.tgz") - if err := c.Build(tmpFile, sourceDir, nil); err != nil { - return "", err - } - - // TODO: can avoid re-reading the file by rewrtiting the Build function and passing io.TeeWriter - data, err := os.OpenFile(tmpFile, os.O_RDONLY, 0) + outputFile, err := os.OpenFile(tmpFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, regularFileMode) if err != nil { return "", err } + defer outputFile.Close() c.hash.Reset() - if _, err := io.Copy(c.hash, data); err != nil { + + output := io.MultiWriter(outputFile, c.hash) + + if err := c.BuildArtefact(tmpFile, sourceDir, output); err != nil { return "", err } - ref := destinationRef + ":" + types.ConfigImageTagPrefix + hex.EncodeToString(c.hash.Sum(nil)) - // _, err := name.ParseReference(ref) - // if err != nil { - // return "", fmt.Errorf("invalid URL: %w", err) - // } + attestLayer, err := c.BuildAttestations(sourceAttestations) + if err != nil { + return "", fmt.Errorf("failed to serialise attestations: %w", err) + } + + ref := destinationRef + ":" + manifestTypes.ConfigImageTagPrefix + hex.EncodeToString(c.hash.Sum(nil)) + tag, err := name.ParseReference(ref) + if err != nil { + return "", fmt.Errorf("invalid URL: %w", err) + } if timestamp == nil { timestamp = new(time.Time) *timestamp = time.Now().UTC() } - img := mutate.Annotations( + indexAnnotations := map[string]string{ + ociclient.CreatedAnnotation: timestamp.Format(time.RFC3339), + } + + index := mutate.Annotations( + empty.Index, + indexAnnotations, + ).(v1.ImageIndex) + + configAnnotations := maps.Clone(indexAnnotations) + + configAnnotations[ContentInterpreterAnnotation] = ContentInterpreterKubectlApply + + config := mutate.Annotations( mutate.ConfigMediaType( - // TODO: define tape media types mutate.MediaType(empty.Image, typesv1.OCIManifestSchema1), - ociclient.CanonicalConfigMediaType, + ConfigMediaType, ), - map[string]string{ - ociclient.CreatedAnnotation: timestamp.Format(time.RFC3339), - }, + configAnnotations, ).(v1.Image) - layer, err := tarball.LayerFromFile(tmpFile, tarball.WithMediaType(ociclient.CanonicalContentMediaType)) + attest := mutate.Annotations( + mutate.ConfigMediaType( + mutate.MediaType(empty.Image, typesv1.OCIManifestSchema1), + ConfigMediaType, + ), + indexAnnotations, + ).(v1.Image) + + // There is an option to use LayerFromReader which will avoid writing any files to disk, + // albeit it might impact memory usage and there is no strict security requirement, and + // manifests do get written out already anyway. + configLayer, err := tarball.LayerFromFile(tmpFile, + tarball.WithMediaType(ContentMediaType), + tarball.WithCompression(compression.GZip), + tarball.WithCompressedCaching, + ) if err != nil { - return "", fmt.Errorf("creating content layer failed: %w", err) + return "", fmt.Errorf("creating artefact content layer failed: %w", err) } - img, err = mutate.Append(img, mutate.Addendum{Layer: layer}) + config, err = mutate.Append(config, mutate.Addendum{Layer: configLayer}) if err != nil { return "", fmt.Errorf("appeding content to artifact failed: %w", err) } - if err := crane.Push(img, ref, c.artefactPushOptions(ctx)...); err != nil { - return "", fmt.Errorf("pushing artifact failed: %w", err) + attest, err = mutate.Append(attest, mutate.Addendum{Layer: attestLayer}) + if err != nil { + return "", fmt.Errorf("appeding attestations to artifact failed: %w", err) } - digest, err := img.Digest() + index = mutate.AppendManifests(index, + mutate.IndexAddendum{Add: config}, + mutate.IndexAddendum{Add: attest}, + ) + digest, err := index.Digest() if err != nil { - return "", fmt.Errorf("parsing artifact digest failed: %w", err) + return "", fmt.Errorf("parsing index digest failed: %w", err) + } + + if err := remote.WriteIndex(tag, index, crane.GetOptions(c.artefactPushOptions(ctx)...).Remote...); err != nil { + return "", fmt.Errorf("pushing index failed: %w", err) } return ref + "@" + digest.String(), err @@ -139,3 +200,124 @@ func (c *Client) artefactPushOptions(ctx context.Context) []crane.Option { OS: "unknown", })) } + +// based on https://github.com/fluxcd/pkg/blob/2a323d771e17af02dee2ccbbb9b445b78ab048e5/oci/client/build.go +func (c *Client) BuildArtefact(artifactPath, + sourceDir string, output io.Writer) error { + absDir, err := filepath.Abs(sourceDir) + if err != nil { + return err + } + + dirStat, err := os.Stat(absDir) + if os.IsNotExist(err) { + return fmt.Errorf("invalid source dir path: %s", absDir) + } + + gw := gzip.NewWriter(output) + tw := tar.NewWriter(gw) + if err := filepath.WalkDir(absDir, func(p string, di os.DirEntry, prevErr error) (err error) { + if prevErr != nil { + return prevErr + } + + // Ignore anything that is not a file or directories e.g. symlinks + ft := di.Type() + if !(ft.IsRegular() || ft.IsDir()) { + return nil + } + + fi, err := di.Info() + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(fi, p) + if err != nil { + return err + } + if dirStat.IsDir() { + // The name needs to be modified to maintain directory structure + // as tar.FileInfoHeader only has access to the base name of the file. + // Ref: https://golang.org/src/archive/tar/common.go?#L6264 + // + // we only want to do this if a directory was passed in + relFilePath, err := filepath.Rel(absDir, p) + if err != nil { + return err + } + // Normalize file path so it works on windows + header.Name = filepath.ToSlash(relFilePath) + } + + // Remove any environment specific data. + header.Gid = 0 + header.Uid = 0 + header.Uname = "" + header.Gname = "" + header.ModTime = time.Time{} + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} + + if err := tw.WriteHeader(header); err != nil { + return err + } + + if !ft.IsRegular() { + return nil + } + + file, err := os.Open(p) + if err != nil { + return err + } + defer ioutil.CheckClose(file, &err) + + if _, err := io.Copy(tw, file); err != nil { + return err + } + return nil + }); err != nil { + _ = tw.Close() + _ = gw.Close() + return err + } + + if err := tw.Close(); err != nil { + _ = gw.Close() + return err + } + if err := gw.Close(); err != nil { + return err + } + + return nil +} + +func (c *Client) BuildAttestations(statements []attestTypes.Statement) (v1.Layer, error) { + output := bytes.NewBuffer(nil) + gw := gzip.NewWriter(output) + + if err := attestTypes.Statements(statements).Encode(gw); err != nil { + return nil, err + } + + if err := gw.Close(); err != nil { + return nil, err + } + + layer, err := tarball.LayerFromOpener( + func() (io.ReadCloser, error) { + // this doesn't copy data, it should re-use same undelying slice + return io.NopCloser(bytes.NewReader(output.Bytes())), nil + }, + tarball.WithMediaType(AttestMediaType), + tarball.WithCompression(compression.GZip), + tarball.WithCompressedCaching, + ) + if err != nil { + return nil, fmt.Errorf("creating attestations layer failed: %w", err) + } + + return layer, nil +} diff --git a/oci/oci.go b/oci/oci.go index 7398cc24..136640d1 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -15,7 +15,9 @@ import ( // OCIv1 "github.com/opencontainers/image-spec/specs-go/v1" ) -const UserAgent = "tape/v1" +const ( + UserAgent = "tape/v1" +) type ( Metadata = ociclient.Metadata diff --git a/tape/app/package.go b/tape/app/package.go index 78f9fe23..79f84357 100644 --- a/tape/app/package.go +++ b/tape/app/package.go @@ -1,10 +1,11 @@ package app import ( + "bytes" "context" + "encoding/base64" "encoding/json" "fmt" - "os" "strings" kimage "sigs.k8s.io/kustomize/api/image" @@ -153,9 +154,13 @@ func (c *TapePackageCommand) Execute(args []string) error { return fmt.Errorf("failed to update manifest files: %w", err) } - if err := attreg.EncodeAllAttestations(os.Stderr); err != nil { - return err - } + c.tape.log.DebugFn(func() []interface{} { + buf := bytes.NewBuffer(nil) + if err := attreg.EncodeAllAttestations(base64.NewEncoder(base64.StdEncoding, buf)); err != nil { + return []interface{}{"failed to encode attestations", err} + } + return []interface{}{"attestations: ", buf.String()} + }) path, sourceEpochTimestamp := loader.MostRecentlyModified() c.tape.log.Debugf("using source epoch timestamp %s from most recently modified manifest file %q", sourceEpochTimestamp, path)