Skip to content

Commit

Permalink
feat: support fallback in push and attach
Browse files Browse the repository at this point in the history 
Signed-off-by: Billy Zha <jinzha1@microsoft.com>
  • Loading branch information
@qweeah
qweeah committed Nov 2, 2022
1 parent b6b1232 commit bb0d8ba
Show file tree
Hide file tree
Showing 2 changed files with 147 additions and 72 deletions.
41 changes: 20 additions & 21 deletions cmd/oras/attach.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,30 +111,33 @@ func runAttach(opts attachOptions) error {
if err != nil {
return err
}
root, err := oras.Pack(
ctx, store, opts.artifactType, descs,
oras.PackOptions{
Subject: &subject,
ManifestAnnotations: annotations[option.AnnotationManifest],
})
if err != nil {
return err
}

// prepare push
packOpts := oras.PackOptions{
Subject: &subject,
ManifestAnnotations: annotations[option.AnnotationManifest],
}
pack := func() (ocispec.Descriptor, error) {
return oras.Pack(ctx, store, opts.artifactType, descs, packOpts)
}

graphCopyOptions := oras.DefaultCopyGraphOptions
graphCopyOptions.Concurrency = opts.concurrency
graphCopyOptions.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if isEqualOCIDescriptor(node, root) {
// skip subject
return descs, nil
updateDisplayOption(&graphCopyOptions, store, opts.Verbose)
copy := func(root ocispec.Descriptor) error {
if root.MediaType == ocispec.MediaTypeArtifactManifest {
graphCopyOptions.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if content.Equal(node, root) {
// skip subject
return descs, nil
}
return content.Successors(ctx, fetcher, node)
}
}
return content.Successors(ctx, fetcher, node)
return oras.CopyGraph(ctx, store, dst, root, graphCopyOptions)
}
updateDisplayOption(&graphCopyOptions, store, opts.Verbose)

// push
err = oras.CopyGraph(ctx, store, dst, root, graphCopyOptions)
root, err := pushArtifact(dst, pack, &packOpts, copy, &graphCopyOptions, opts.Verbose)
if err != nil {
return err
}
Expand All @@ -145,7 +148,3 @@ func runAttach(opts attachOptions) error {
// Export manifest
return opts.ExportManifest(ctx, store, root)
}

func isEqualOCIDescriptor(a, b ocispec.Descriptor) bool {
return a.Size == b.Size && a.Digest == b.Digest && a.MediaType == b.MediaType
}
178 changes: 127 additions & 51 deletions cmd/oras/push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ package main

import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"sync"

Expand All @@ -27,14 +29,12 @@ import (
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/errcode"
"oras.land/oras/cmd/oras/internal/display"
"oras.land/oras/cmd/oras/internal/option"
)

const (
tagStaged = "staged"
)

type pushOptions struct {
option.Common
option.Remote
Expand Down Expand Up @@ -120,98 +120,174 @@ func runPush(opts pushOptions) error {
return err
}

// Prepare manifest
// prepare pack
packOpts := oras.PackOptions{
ConfigAnnotations: annotations[option.AnnotationConfig],
ManifestAnnotations: annotations[option.AnnotationManifest],
}
store := file.New("")
defer store.Close()
store.AllowPathTraversalOnWrite = opts.PathValidationDisabled

// Ready to push
copyOptions := oras.DefaultCopyOptions
copyOptions.Concurrency = opts.concurrency
updateDisplayOption(&copyOptions.CopyGraphOptions, store, opts.Verbose)
desc, err := packManifest(ctx, store, annotations, &opts)
if opts.manifestConfigRef != "" {
path, cfgMediaType := parseFileReference(opts.manifestConfigRef, oras.MediaTypeUnknownConfig)
desc, err := store.Add(ctx, option.AnnotationConfig, cfgMediaType, path)
if err != nil {
return err
}
desc.Annotations = packOpts.ConfigAnnotations
packOpts.ConfigDescriptor = &desc
packOpts.PackImageManifest = true
}
descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose)
if err != nil {
return err
}
pack := func() (ocispec.Descriptor, error) {
root, err := oras.Pack(ctx, store, opts.artifactType, descs, packOpts)
if err != nil {
return ocispec.Descriptor{}, err
}
if err = store.Tag(ctx, root, root.Digest.String()); err != nil {
return ocispec.Descriptor{}, err
}
return root, nil
}

// Push
// prepare push
dst, err := opts.NewRepository(opts.targetRef, opts.Common)
if err != nil {
return err
}
if tag := dst.Reference.Reference; tag == "" {
err = oras.CopyGraph(ctx, store, dst, desc, copyOptions.CopyGraphOptions)
} else {
desc, err = oras.Copy(ctx, store, tagStaged, dst, tag, copyOptions)
copyOptions := oras.DefaultCopyOptions
copyOptions.Concurrency = opts.concurrency
updateDisplayOption(&copyOptions.CopyGraphOptions, store, opts.Verbose)
copy := func(root ocispec.Descriptor) error {
if tag := dst.Reference.Reference; tag == "" {
err = oras.CopyGraph(ctx, store, dst, root, copyOptions.CopyGraphOptions)
} else {
_, err = oras.Copy(ctx, store, root.Digest.String(), dst, tag, copyOptions)
}
return err
}

// Push
root, err := pushArtifact(dst, pack, &packOpts, copy, &copyOptions.CopyGraphOptions, opts.Verbose)
if err != nil {
return err
}

fmt.Println("Pushed", opts.targetRef)

if len(opts.extraRefs) != 0 {
contentBytes, err := content.FetchAll(ctx, store, desc)
contentBytes, err := content.FetchAll(ctx, store, root)
if err != nil {
return err
}
tagBytesNOpts := oras.DefaultTagBytesNOptions
tagBytesNOpts.Concurrency = opts.concurrency
if _, err = oras.TagBytesN(ctx, &display.TagManifestStatusPrinter{Repository: dst}, desc.MediaType, contentBytes, opts.extraRefs, tagBytesNOpts); err != nil {
if _, err = oras.TagBytesN(ctx, &display.TagManifestStatusPrinter{Repository: dst}, root.MediaType, contentBytes, opts.extraRefs, tagBytesNOpts); err != nil {
return err
}
}

fmt.Println("Digest:", desc.Digest)
fmt.Println("Digest:", root.Digest)

// Export manifest
return opts.ExportManifest(ctx, store, desc)
return opts.ExportManifest(ctx, store, root)
}

func packManifest(ctx context.Context, store *file.Store, annotations map[string]map[string]string, opts *pushOptions) (ocispec.Descriptor, error) {
var packOpts oras.PackOptions
packOpts.ConfigAnnotations = annotations[option.AnnotationConfig]
packOpts.ManifestAnnotations = annotations[option.AnnotationManifest]

if opts.manifestConfigRef != "" {
path, mediatype := parseFileReference(opts.manifestConfigRef, oras.MediaTypeUnknownConfig)
desc, err := store.Add(ctx, option.AnnotationConfig, mediatype, path)
if err != nil {
return ocispec.Descriptor{}, err
func updateDisplayOption(opts *oras.CopyGraphOptions, store content.Fetcher, verbose bool) {
committed := &sync.Map{}
opts.PreCopy = display.StatusPrinter("Uploading", verbose)
opts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
return display.PrintStatus(desc, "Exists ", verbose)
}
opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, verbose); err != nil {
return err
}
desc.Annotations = packOpts.ConfigAnnotations
packOpts.ConfigDescriptor = &desc
packOpts.PackImageManifest = true
return display.PrintStatus(desc, "Uploaded ", verbose)
}
descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose)
}

type packFunc func() (ocispec.Descriptor, error)
type copyFunc func(desc ocispec.Descriptor) error

func pushArtifact(dst *remote.Repository, pack packFunc, packOpts *oras.PackOptions, copy copyFunc, copyOpts *oras.CopyGraphOptions, verbose bool) (ocispec.Descriptor, error) {
root, err := pack()
if err != nil {
return ocispec.Descriptor{}, err
}

// pack artifact
manifestDesc, err := oras.Pack(ctx, store, opts.artifactType, descs, packOpts)
copyRootAttempted := false
preCopy := copyOpts.PreCopy
copyOpts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
if content.Equal(root, desc) {
copyRootAttempted = true
}
return preCopy(ctx, desc)
}

// push
if err = copy(root); err == nil {
return root, nil
}

if !copyRootAttempted || !isArtifactUnsupported(err) {
return ocispec.Descriptor{}, err
}

if err := display.PrintStatus(root, "Fallback ", verbose); err != nil {
return ocispec.Descriptor{}, err
}
dst.SetReferrersCapability(false)
packOpts.PackImageManifest = true
root, err = pack()
if err != nil {
return ocispec.Descriptor{}, err
}

if err = store.Tag(ctx, manifestDesc, tagStaged); err != nil {
copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if content.Equal(node, root) {
// skip non-config
content, err := content.FetchAll(ctx, fetcher, root)
if err != nil {
return nil, err
}
var manifest ocispec.Manifest
if err := json.Unmarshal(content, &manifest); err != nil {
return nil, err
}
return []ocispec.Descriptor{manifest.Config}, nil
}

// config has no successors
return nil, nil
}
err = copy(root)
if err != nil {
return ocispec.Descriptor{}, err
}
return manifestDesc, nil
return root, nil
}

func updateDisplayOption(opts *oras.CopyGraphOptions, store content.Fetcher, verbose bool) {
committed := &sync.Map{}
opts.PreCopy = display.StatusPrinter("Uploading", verbose)
opts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
return display.PrintStatus(desc, "Exists ", verbose)
func isArtifactUnsupported(err error) bool {
var errResp *errcode.ErrorResponse
if !errors.As(err, &errResp) || errResp.StatusCode != http.StatusBadRequest {
return false
}
opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, verbose); err != nil {
return err
}
return display.PrintStatus(desc, "Uploaded ", verbose)

var errCode errcode.Error
if !errors.As(errResp, &errCode) {
return false
}

// As of November 2022, ECR is known to return UNSUPPORTED error when
// putting an OCI artifact manifest.
switch errCode.Code {
case errcode.ErrorCodeManifestInvalid, errcode.ErrorCodeUnsupported:
return true
}
return false
}

0 comments on commit bb0d8ba

Please sign in to comment.