Bump oras to v2 (#126)

* Bump oras-go to v2 * Update build to oras v2 Signed-off-by: carabasdaniel <dani@aserto.com> * Fix push & tag * Allow rebuild Signed-off-by: carabasdaniel <dani@aserto.com> * Fix build images * Oci Tag only on not resolved Signed-off-by: carabasdaniel <dani@aserto.com> * PostCopy tag Signed-off-by: carabasdaniel <dani@aserto.com> * Add copyskipped Signed-off-by: carabasdaniel <dani@aserto.com> * Add media type check for copy options Signed-off-by: carabasdaniel <dani@aserto.com> * Extract remote manager Signed-off-by: carabasdaniel <dani@aserto.com> * Test with pre-release Signed-off-by: carabasdaniel <dani@aserto.com> * Fix on rebase Signed-off-by: carabasdaniel <dani@aserto.com> * Use oci package in repl Signed-off-by: carabasdaniel <dani@aserto.com> * Hack to allow rm to work with oras v2 until oras-project/oras-go#454 Signed-off-by: carabasdaniel <dani@aserto.com> * Use oras v2 in save command Signed-off-by: carabasdaniel <dani@aserto.com> * Use oras v2 in inspect Signed-off-by: carabasdaniel <dani@aserto.com> * Fix linting errors Signed-off-by: carabasdaniel <dani@aserto.com> --------- Signed-off-by: carabasdaniel <dani@aserto.com> Co-authored-by: oanatmaria <oana@aserto.com>
opcr-io · Mar 15, 2023 · eeeedda · eeeedda
1 parent dff729f
commit eeeedda
Show file tree

Hide file tree

Showing 11 changed files with 376 additions and 233 deletions.
diff --git a/go.mod b/go.mod
@@ -14,7 +14,6 @@ require (
 	github.com/containerd/containerd v1.6.15
 	github.com/dustin/go-humanize v1.0.0
 	github.com/golang/mock v1.6.0
-	github.com/google/uuid v1.3.0
 	github.com/google/wire v0.5.0
 	github.com/magefile/mage v1.14.0
 	github.com/mitchellh/mapstructure v1.5.0
@@ -29,7 +28,7 @@ require (
 	golang.org/x/term v0.4.0
 	google.golang.org/grpc v1.51.0
 	gopkg.in/yaml.v2 v2.4.0
-	oras.land/oras-go v1.2.2
+	oras.land/oras-go/v2 v2.0.0
 	sigs.k8s.io/controller-runtime v0.14.1
 )
 
@@ -66,6 +65,7 @@ require (
 	github.com/google/go-github/v33 v33.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/subcommands v1.0.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.14.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -133,4 +133,5 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect
+	oras.land/oras-go v1.2.2 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -859,6 +859,8 @@ k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5
 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.2 h1:0E9tOHUfrNH7TCDk5KU0jVBEzCqbfdyuVfGmJ7ZeRPE=
 oras.land/oras-go v1.2.2/go.mod h1:Apa81sKoZPpP7CDciE006tSZ0x3Q3+dOoBcMZ/aNxvw=
+oras.land/oras-go/v2 v2.0.0 h1:+LRAz92WF7AvYQsQjPEAIw3Xb2zPPhuydjpi4pIHmc0=
+oras.land/oras-go/v2 v2.0.0/go.mod h1:iVExH1NxrccIxjsiq17L91WCZ4KIw6jVQyCLsZsu1gc=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

diff --git a/go.work.sum b/go.work.sum
@@ -393,7 +393,6 @@ google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6F
 google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
 google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
 google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70=
-google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I=
 google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=

diff --git a/pkg/app/build.go b/pkg/app/build.go
@@ -1,22 +1,22 @@
 package app
 
 import (
-	"io"
+	"bufio"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/aserto-dev/runtime"
-	containerd_content "github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/reference/docker"
-	"github.com/google/uuid"
+
 	"github.com/opcr-io/policy/pkg/oci"
 	"github.com/opcr-io/policy/pkg/parser"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
-	"oras.land/oras-go/pkg/content"
+	orasoci "oras.land/oras-go/v2/content/oci"
 )
 
 const (
@@ -84,11 +84,7 @@ func (c *PolicyApp) Build(ref string, path []string, annotations map[string]stri
 		return errors.Wrap(err, "failed to build opa policy bundle")
 	}
 
-	ociStore, err := content.NewOCI(c.Configuration.PoliciesRoot())
-	if err != nil {
-		return err
-	}
-	err = ociStore.LoadIndex()
+	ociStore, err := orasoci.New(c.Configuration.PoliciesRoot())
 	if err != nil {
 		return err
 	}
@@ -97,27 +93,31 @@ func (c *PolicyApp) Build(ref string, path []string, annotations map[string]stri
 		annotations = map[string]string{}
 	}
 
-	parsedRef, err := docker.ParseDockerRef(ref)
+	familiarezedRef, err := parser.CalculatePolicyRef(ref, c.Configuration.DefaultDomain)
+	if err != nil {
+		return errors.Wrap(err, "failed to calculate policy reference")
+	}
+
+	parsedRef, err := docker.ParseDockerRef(familiarezedRef)
 	if err != nil {
 		return err
 	}
+
 	annotations[ocispec.AnnotationTitle] = docker.TrimNamed(parsedRef).String()
 	annotations[AnnotationPolicyRegistryType] = PolicyTypePolicy
 	annotations[ocispec.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339)
 
-	descriptor, err := c.createImage(ociStore, outfile, annotations)
+	desc, err := c.createImage(ociStore, outfile, annotations)
 	if err != nil {
 		return err
 	}
 
-	parsed, err := parser.CalculatePolicyRef(ref, c.Configuration.DefaultDomain)
+	err = ociStore.Tag(c.Context, desc, parsedRef.String())
 	if err != nil {
-		return errors.Wrap(err, "failed to calculate policy reference")
+		return err
 	}
 
-	ociStore.AddReference(parsed, descriptor)
-
-	c.UI.Normal().WithStringValue("reference", ref).Msg("Tagging image.")
+	c.UI.Normal().WithStringValue("reference", parsedRef.String()).Msg("Tagging image.")
 
 	err = ociStore.SaveIndex()
 	if err != nil {
@@ -127,27 +127,19 @@ func (c *PolicyApp) Build(ref string, path []string, annotations map[string]stri
 	return nil
 }
 
-func (c *PolicyApp) createImage(ociStore *content.OCI, tarball string, annotations map[string]string) (ocispec.Descriptor, error) {
+func (c *PolicyApp) createImage(ociStore *orasoci.Store, tarball string, annotations map[string]string) (ocispec.Descriptor, error) {
 	descriptor := ocispec.Descriptor{}
-
+	ociStore.AutoSaveIndex = true
 	fDigest, err := c.fileDigest(tarball)
 	if err != nil {
 		return descriptor, err
 	}
 
-	_, err = ociStore.Info(c.Context, fDigest)
-	if err != nil && !errors.Is(err, errdefs.ErrNotFound) {
+	tarballFile, err := os.Open(tarball)
+	if err != nil {
 		return descriptor, err
 	}
-
-	if err == nil {
-		err = ociStore.Delete(c.Context, fDigest)
-		if err != nil {
-			return descriptor, errors.Wrap(err, "couldn't overwrite existing image")
-		}
-	}
-
-	tarballFile, err := os.Open(tarball)
+	fileInfo, err := tarballFile.Stat()
 	if err != nil {
 		return descriptor, err
 	}
@@ -158,44 +150,35 @@ func (c *PolicyApp) createImage(ociStore *content.OCI, tarball string, annotatio
 		}
 	}()
 
-	fileInfo, err := tarballFile.Stat()
-	if err != nil {
-		return descriptor, err
-	}
-
-	descriptor = ocispec.Descriptor{
-		MediaType:   oci.MediaTypeImageLayer,
-		Digest:      fDigest,
-		Size:        fileInfo.Size(),
-		Annotations: annotations,
-	}
+	descriptor.Digest = fDigest
+	descriptor.Size = fileInfo.Size()
+	descriptor.Annotations = annotations
+	descriptor.MediaType = oci.MediaTypeImageLayer
 
-	ociWriter, err := ociStore.Writer(
-		c.Context,
-		containerd_content.WithDescriptor(descriptor),
-		containerd_content.WithRef(uuid.NewString()))
-	if err != nil {
+	exists, err := ociStore.Exists(c.Context, descriptor)
+	if err != nil && !errors.Is(err, errdefs.ErrNotFound) {
 		return descriptor, err
 	}
-	defer func() {
-		err := ociWriter.Close()
+
+	if exists {
+		// Hack to remove the existing digest until ocistore deleter is implemented
+		// https://github.com/oras-project/oras-go/issues/454
+		digestPath := filepath.Join(strings.Split(descriptor.Digest.String(), ":")...)
+		blob := filepath.Join(c.Configuration.PoliciesRoot(), "blobs", digestPath)
+		err = os.Remove(blob)
 		if err != nil {
-			c.UI.Problem().WithErr(err).Msg("Failed to close local OCI store.")
+			return descriptor, err
 		}
-	}()
-
-	_, err = io.Copy(ociWriter, tarballFile)
-	if err != nil {
-		return descriptor, err
 	}
 
-	err = ociWriter.Commit(c.Context, fileInfo.Size(), fDigest)
+	reader := bufio.NewReader(tarballFile)
+
+	err = ociStore.Push(c.Context, descriptor, reader)
 	if err != nil {
 		return descriptor, err
 	}
-
 	c.UI.Normal().
-		WithStringValue("digest", ociWriter.Digest().String()).
+		WithStringValue("digest", descriptor.Digest.String()).
 		Msg("Created new image.")
 
 	return descriptor, nil

diff --git a/pkg/app/images.go b/pkg/app/images.go
@@ -3,38 +3,39 @@ package app
 import (
 	"sort"
 	"strings"
-	"time"
 
 	"github.com/containerd/containerd/reference/docker"
 	"github.com/dustin/go-humanize"
-	"oras.land/oras-go/pkg/content"
+	"oras.land/oras-go/v2/content/oci"
 )
 
 func (c *PolicyApp) Images() error {
 	defer c.Cancel()
 
 	var data [][]string
 
-	ociStore, err := content.NewOCI(c.Configuration.PoliciesRoot())
+	ociStore, err := oci.New(c.Configuration.PoliciesRoot())
 	if err != nil {
 		return err
 	}
 
-	err = ociStore.LoadIndex()
-	if err != nil {
+	table := c.UI.Normal().WithTable("Repository", "Tag", "Image ID", "Size")
+	var tgs []string
+	err = ociStore.Tags(c.Context, "", func(tags []string) error {
+		tgs = append(tgs, tags...)
 		return nil
+	})
+	if err != nil {
+		return err
 	}
 
-	table := c.UI.Normal().WithTable("Repository", "Tag", "Image ID", "Created", "Size")
-	refs := ociStore.ListReferences()
-
-	for k, v := range refs {
-		info, err := ociStore.Info(c.Context, v.Digest)
+	for _, tag := range tgs {
+		descr, err := ociStore.Resolve(c.Context, tag)
 		if err != nil {
 			return err
 		}
 
-		ref, err := docker.ParseDockerRef(k)
+		ref, err := docker.ParseDockerRef(tag)
 		if err != nil {
 			return err
 		}
@@ -55,22 +56,20 @@ func (c *PolicyApp) Images() error {
 		arrData := []string{
 			familiarName,
 			tagOrNone,
-			info.Digest.Encoded()[:12],
-			humanize.Time(info.CreatedAt),
-			strings.ReplaceAll(humanize.Bytes(uint64(v.Size)), " ", ""),
-			info.CreatedAt.Format(time.RFC3339Nano)}
+			descr.Digest.Encoded()[:12],
+			strings.ReplaceAll(humanize.Bytes(uint64(descr.Size)), " ", "")}
 
 		data = append(data, arrData)
 	}
 
 	// sort data by CreatedAt DESC.
 	sort.SliceStable(data, func(i, j int) bool {
-		return data[i][5] < data[j][5] || (data[i][5] == data[j][5] && data[i][1] < data[j][1])
+		return data[i][3] < data[j][3] || (data[i][3] == data[j][3] && data[i][1] < data[j][1])
 	})
 
 	for i := len(data) - 1; i >= 0; i-- {
 		v := data[i]
-		table.WithTableRow(v[0], v[1], v[2], v[3], v[4])
+		table.WithTableRow(v[0], v[1], v[2], v[3])
 	}
 
 	table.Do()

diff --git a/pkg/app/inspect.go b/pkg/app/inspect.go
@@ -1,9 +1,9 @@
 package app
 
 import (
+	"github.com/opcr-io/policy/pkg/oci"
 	"github.com/opcr-io/policy/pkg/parser"
 	"github.com/pkg/errors"
-	"oras.land/oras-go/pkg/content"
 )
 
 func (c *PolicyApp) Inspect(userRef string) error {
@@ -14,53 +14,30 @@ func (c *PolicyApp) Inspect(userRef string) error {
 		return err
 	}
 
-	ociStore, err := content.NewOCI(c.Configuration.PoliciesRoot())
+	ociClient, err := oci.NewOCI(c.Context, c.Logger, c.getHosts, c.Configuration.PoliciesRoot())
 	if err != nil {
 		return err
 	}
-	err = ociStore.LoadIndex()
-	if err != nil {
-		return err
-	}
-
-	refs := ociStore.ListReferences()
 
-	refDescriptor, ok := refs[ref]
-	if !ok {
-		return errors.Errorf("policy [%s] not found in the local store", ref)
-	}
-
-	contentInfo, err := ociStore.Info(c.Context, refDescriptor.Digest)
+	contentInfo, err := ociClient.GetStore().Resolve(c.Context, ref)
 	if err != nil {
 		return errors.Wrapf(err, "failed to read content info for policy [%s]", ref)
 	}
 
 	c.UI.Normal().
+		WithStringValue("media type", contentInfo.MediaType).
 		WithStringValue("digest", contentInfo.Digest.String()).
 		WithIntValue("size", contentInfo.Size).
-		WithStringValue("created_at", contentInfo.CreatedAt.String()).
-		WithStringValue("updated_at", contentInfo.UpdatedAt.String()).
 		Do()
 
-	if len(refDescriptor.Annotations) > 0 {
+	if len(contentInfo.Annotations) > 0 {
 		msg := c.UI.Normal().WithTable("Annotation", "Value")
 
-		for k, v := range refDescriptor.Annotations {
+		for k, v := range contentInfo.Annotations {
 			msg.WithTableRow(k, v)
 		}
-
 		msg.Msg("Annotations")
 	}
 
-	if len(contentInfo.Labels) > 0 {
-		msg := c.UI.Normal().WithTable("Label", "Value")
-
-		for k, v := range contentInfo.Labels {
-			msg.WithTableRow(k, v)
-		}
-
-		msg.Msg("Labels")
-	}
-
 	return nil
 }