Merge pull request #5 from docker/bundle-attestations

Bundle attestations

oci/artefact.go

@@ -1,23 +1,46 @@
 package oci
 
 import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
+	"maps"
 	"os"
 	"path/filepath"
 	"time"
 
 	ociclient "github.com/fluxcd/pkg/oci"
+	"github.com/go-git/go-git/v5/utils/ioutil"
+	"github.com/google/go-containerregistry/pkg/compression"
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	typesv1 "github.com/google/go-containerregistry/pkg/v1/types"
 
-	"github.com/docker/labs-brown-tape/manifest/types"
+	attestTypes "github.com/docker/labs-brown-tape/attest/types"
+	manifestTypes "github.com/docker/labs-brown-tape/manifest/types"
+)
+
+const (
+	mediaTypePrefix  = "application/vnd.docker.tape"
+	ConfigMediaType  = mediaTypePrefix + ".config.v1alpha1+json"
+	ContentMediaType = mediaTypePrefix + ".content.v1alpha1.tar+gzip"
+	AttestMediaType  = mediaTypePrefix + ".attest.v1alpha1.jsonl+gzip"
+
+	ContentInterpreterAnnotation   = mediaTypePrefix + "content-interpreter.v1alpha1"
+	ContentInterpreterKubectlApply = mediaTypePrefix + ".kubectl-apply.v1alpha1.tar+gzip"
+
+	// TODO: content inteprete invocation with an image
+
+	regularFileMode = 0o640
 )
 
 type ArtefactInfo struct {
@@ -64,7 +87,7 @@ func (c *Client) GetArtefact(ctx context.Context, ref string) (*ArtefactInfo, er
 }
 
 // based on https://github.com/fluxcd/pkg/blob/2a323d771e17af02dee2ccbbb9b445b78ab048e5/oci/client/push.go
-func (c *Client) PushArtefact(ctx context.Context, destinationRef, sourceDir string, timestamp *time.Time) (string, error) {
+func (c *Client) PushArtefact(ctx context.Context, destinationRef, sourceDir string, timestamp *time.Time, sourceAttestations ...attestTypes.Statement) (string, error) {
 	tmpDir, err := os.MkdirTemp("", "bpt-oci-artefact-*")
 	if err != nil {
 		return "", err
@@ -73,60 +96,98 @@ func (c *Client) PushArtefact(ctx context.Context, destinationRef, sourceDir str
 
 	tmpFile := filepath.Join(tmpDir, "artefact.tgz")
 
-	if err := c.Build(tmpFile, sourceDir, nil); err != nil {
-		return "", err
-	}
-
-	// TODO: can avoid re-reading the file by rewrtiting the Build function and passing io.TeeWriter
-	data, err := os.OpenFile(tmpFile, os.O_RDONLY, 0)
+	outputFile, err := os.OpenFile(tmpFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, regularFileMode)
 	if err != nil {
 		return "", err
 	}
+	defer outputFile.Close()
 
 	c.hash.Reset()
-	if _, err := io.Copy(c.hash, data); err != nil {
+
+	output := io.MultiWriter(outputFile, c.hash)
+
+	if err := c.BuildArtefact(tmpFile, sourceDir, output); err != nil {
 		return "", err
 	}
 
-	ref := destinationRef + ":" + types.ConfigImageTagPrefix + hex.EncodeToString(c.hash.Sum(nil))
-	// _, err := name.ParseReference(ref)
-	// if err != nil {
-	// 	return "", fmt.Errorf("invalid URL: %w", err)
-	// }
+	attestLayer, err := c.BuildAttestations(sourceAttestations)
+	if err != nil {
+		return "", fmt.Errorf("failed to serialise attestations: %w", err)
+	}
+
+	ref := destinationRef + ":" + manifestTypes.ConfigImageTagPrefix + hex.EncodeToString(c.hash.Sum(nil))
+	tag, err := name.ParseReference(ref)
+	if err != nil {
+		return "", fmt.Errorf("invalid URL: %w", err)
+	}
 
 	if timestamp == nil {
 		timestamp = new(time.Time)
 		*timestamp = time.Now().UTC()
 	}
 
-	img := mutate.Annotations(
+	indexAnnotations := map[string]string{
+		ociclient.CreatedAnnotation: timestamp.Format(time.RFC3339),
+	}
+
+	index := mutate.Annotations(
+		empty.Index,
+		indexAnnotations,
+	).(v1.ImageIndex)
+
+	configAnnotations := maps.Clone(indexAnnotations)
+
+	configAnnotations[ContentInterpreterAnnotation] = ContentInterpreterKubectlApply
+
+	config := mutate.Annotations(
 		mutate.ConfigMediaType(
-			// TODO: define tape media types
 			mutate.MediaType(empty.Image, typesv1.OCIManifestSchema1),
-			ociclient.CanonicalConfigMediaType,
+			ConfigMediaType,
 		),
-		map[string]string{
-			ociclient.CreatedAnnotation: timestamp.Format(time.RFC3339),
-		},
+		configAnnotations,
 	).(v1.Image)
 
-	layer, err := tarball.LayerFromFile(tmpFile, tarball.WithMediaType(ociclient.CanonicalContentMediaType))
+	attest := mutate.Annotations(
+		mutate.ConfigMediaType(
+			mutate.MediaType(empty.Image, typesv1.OCIManifestSchema1),
+			ConfigMediaType,
+		),
+		indexAnnotations,
+	).(v1.Image)
+
+	// There is an option to use LayerFromReader which will avoid writing any files to disk,
+	// albeit it might impact memory usage and there is no strict security requirement, and
+	// manifests do get written out already anyway.
+	configLayer, err := tarball.LayerFromFile(tmpFile,
+		tarball.WithMediaType(ContentMediaType),
+		tarball.WithCompression(compression.GZip),
+		tarball.WithCompressedCaching,
+	)
 	if err != nil {
-		return "", fmt.Errorf("creating content layer failed: %w", err)
+		return "", fmt.Errorf("creating artefact content layer failed: %w", err)
 	}
 
-	img, err = mutate.Append(img, mutate.Addendum{Layer: layer})
+	config, err = mutate.Append(config, mutate.Addendum{Layer: configLayer})
 	if err != nil {
 		return "", fmt.Errorf("appeding content to artifact failed: %w", err)
 	}
 
-	if err := crane.Push(img, ref, c.artefactPushOptions(ctx)...); err != nil {
-		return "", fmt.Errorf("pushing artifact failed: %w", err)
+	attest, err = mutate.Append(attest, mutate.Addendum{Layer: attestLayer})
+	if err != nil {
+		return "", fmt.Errorf("appeding attestations to artifact failed: %w", err)
 	}
 
-	digest, err := img.Digest()
+	index = mutate.AppendManifests(index,
+		mutate.IndexAddendum{Add: config},
+		mutate.IndexAddendum{Add: attest},
+	)
+	digest, err := index.Digest()
 	if err != nil {
-		return "", fmt.Errorf("parsing artifact digest failed: %w", err)
+		return "", fmt.Errorf("parsing index digest failed: %w", err)
+	}
+
+	if err := remote.WriteIndex(tag, index, crane.GetOptions(c.artefactPushOptions(ctx)...).Remote...); err != nil {
+		return "", fmt.Errorf("pushing index failed: %w", err)
 	}
 
 	return ref + "@" + digest.String(), err
@@ -139,3 +200,124 @@ func (c *Client) artefactPushOptions(ctx context.Context) []crane.Option {
 			OS:           "unknown",
 		}))
 }
+
+// based on https://github.com/fluxcd/pkg/blob/2a323d771e17af02dee2ccbbb9b445b78ab048e5/oci/client/build.go
+func (c *Client) BuildArtefact(artifactPath,
+	sourceDir string, output io.Writer) error {
+	absDir, err := filepath.Abs(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	dirStat, err := os.Stat(absDir)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("invalid source dir path: %s", absDir)
+	}
+
+	gw := gzip.NewWriter(output)
+	tw := tar.NewWriter(gw)
+	if err := filepath.WalkDir(absDir, func(p string, di os.DirEntry, prevErr error) (err error) {
+		if prevErr != nil {
+			return prevErr
+		}
+
+		// Ignore anything that is not a file or directories e.g. symlinks
+		ft := di.Type()
+		if !(ft.IsRegular() || ft.IsDir()) {
+			return nil
+		}
+
+		fi, err := di.Info()
+		if err != nil {
+			return err
+		}
+
+		header, err := tar.FileInfoHeader(fi, p)
+		if err != nil {
+			return err
+		}
+		if dirStat.IsDir() {
+			// The name needs to be modified to maintain directory structure
+			// as tar.FileInfoHeader only has access to the base name of the file.
+			// Ref: https://golang.org/src/archive/tar/common.go?#L6264
+			//
+			// we only want to do this if a directory was passed in
+			relFilePath, err := filepath.Rel(absDir, p)
+			if err != nil {
+				return err
+			}
+			// Normalize file path so it works on windows
+			header.Name = filepath.ToSlash(relFilePath)
+		}
+
+		// Remove any environment specific data.
+		header.Gid = 0
+		header.Uid = 0
+		header.Uname = ""
+		header.Gname = ""
+		header.ModTime = time.Time{}
+		header.AccessTime = time.Time{}
+		header.ChangeTime = time.Time{}
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		if !ft.IsRegular() {
+			return nil
+		}
+
+		file, err := os.Open(p)
+		if err != nil {
+			return err
+		}
+		defer ioutil.CheckClose(file, &err)
+
+		if _, err := io.Copy(tw, file); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		_ = tw.Close()
+		_ = gw.Close()
+		return err
+	}
+
+	if err := tw.Close(); err != nil {
+		_ = gw.Close()
+		return err
+	}
+	if err := gw.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) BuildAttestations(statements []attestTypes.Statement) (v1.Layer, error) {
+	output := bytes.NewBuffer(nil)
+	gw := gzip.NewWriter(output)
+
+	if err := attestTypes.Statements(statements).Encode(gw); err != nil {
+		return nil, err
+	}
+
+	if err := gw.Close(); err != nil {
+		return nil, err
+	}
+
+	layer, err := tarball.LayerFromOpener(
+		func() (io.ReadCloser, error) {
+			// this doesn't copy data, it should re-use same undelying slice
+			return io.NopCloser(bytes.NewReader(output.Bytes())), nil
+		},
+		tarball.WithMediaType(AttestMediaType),
+		tarball.WithCompression(compression.GZip),
+		tarball.WithCompressedCaching,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("creating attestations layer failed: %w", err)
+	}
+
+	return layer, nil
+}

oci/oci.go

@@ -15,7 +15,9 @@ import (
 	// OCIv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-const UserAgent = "tape/v1"
+const (
+	UserAgent = "tape/v1"
+)
 
 type (
 	Metadata   = ociclient.Metadata

tape/app/package.go

@@ -1,10 +1,11 @@
 package app
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"os"
 	"strings"
 
 	kimage "sigs.k8s.io/kustomize/api/image"
@@ -153,9 +154,13 @@ func (c *TapePackageCommand) Execute(args []string) error {
 		return fmt.Errorf("failed to update manifest files: %w", err)
 	}
 
-	if err := attreg.EncodeAllAttestations(os.Stderr); err != nil {
-		return err
-	}
+	c.tape.log.DebugFn(func() []interface{} {
+		buf := bytes.NewBuffer(nil)
+		if err := attreg.EncodeAllAttestations(base64.NewEncoder(base64.StdEncoding, buf)); err != nil {
+			return []interface{}{"failed to encode attestations", err}
+		}
+		return []interface{}{"attestations: ", buf.String()}
+	})
 
 	path, sourceEpochTimestamp := loader.MostRecentlyModified()
 	c.tape.log.Debugf("using source epoch timestamp %s from most recently modified manifest file %q", sourceEpochTimestamp, path)