Skip to content

Commit

Permalink
feat(sync): sync references(signatures/artifacts) recursively
Browse files Browse the repository at this point in the history 
sync now also pulls chained artifacts recursively
eg:
 image->sbom->sbom signature
 image->artifact->artifact

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
  • Loading branch information
@eusebiu-constantin-petu-dbk
eusebiu-constantin-petu-dbk committed Jun 15, 2023
1 parent 03f47f6 commit 7321edf
Show file tree
Hide file tree
Showing 11 changed files with 446 additions and 221 deletions.
25 changes: 3 additions & 22 deletions pkg/api/routes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/sigstore/cosign/v2/pkg/oci/remote"

zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/constants"
Expand Down Expand Up @@ -1734,33 +1733,15 @@ func getImageManifest(routeHandler *RouteHandler, imgStore storageTypes.ImageSto
routeHandler.c.Log.Info().Str("repository", name).Str("reference", reference).
Msg("trying to get updated image by syncing on demand")

// we use a custom method for syncing cosign signatures for the moment, even though it's also an oci image.
if isCosignTag(reference) {
if errSync := routeHandler.c.SyncOnDemand.SyncReference(name, reference, syncConstants.Cosign); errSync != nil {
routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", reference).
Msg("error encounter while syncing cosign signature for image")
}
} else {
if errSync := routeHandler.c.SyncOnDemand.SyncImage(name, reference); errSync != nil {
routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", reference).
Msg("error encounter while syncing image")
}
if errSync := routeHandler.c.SyncOnDemand.SyncImage(name, reference); errSync != nil {
routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", reference).
Msg("error encounter while syncing image")
}
}

return imgStore.GetImageManifest(name, reference)
}

// this function will check if tag is a cosign tag (signature or sbom).
func isCosignTag(tag string) bool {
if strings.HasPrefix(tag, "sha256-") &&
(strings.HasSuffix(tag, remote.SignatureTagSuffix) || strings.HasSuffix(tag, remote.SBOMTagSuffix)) {
return true
}

return false
}

// will sync referrers on demand if they are not found, in case sync extensions is enabled.
func getOrasReferrers(routeHandler *RouteHandler,
imgStore storageTypes.ImageStore, name string, digest godigest.Digest,
Expand Down
17 changes: 3 additions & 14 deletions pkg/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"unicode/utf8"

"github.com/gorilla/mux"
"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"

"zotregistry.io/zot/pkg/log"
Expand All @@ -37,9 +36,9 @@ func AllowedMethods(methods ...string) []string {
return append(methods, http.MethodOptions)
}

func Contains(slice []string, item string) bool {
for _, v := range slice {
if item == v {
func Contains[T comparable](elems []T, v T) bool {
for _, s := range elems {
if v == s {
return true
}
}
Expand Down Expand Up @@ -271,16 +270,6 @@ func MarshalThroughStruct(obj interface{}, throughStruct interface{}) ([]byte, e
return toJSON, nil
}

func DContains(slice []digest.Digest, item digest.Digest) bool {
for _, v := range slice {
if item == v {
return true
}
}

return false
}

func GetManifestArtifactType(manifestContent ispec.Manifest) string {
if manifestContent.ArtifactType != "" {
return manifestContent.ArtifactType
Expand Down
91 changes: 39 additions & 52 deletions pkg/extensions/sync/references/cosign.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
package references

import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"

godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sigstore/cosign/v2/pkg/oci/remote"

Expand Down Expand Up @@ -46,77 +46,70 @@ func (ref CosignReference) Name() string {

func (ref CosignReference) IsSigned(upstreamRepo, subjectDigestStr string) bool {
cosignSignatureTag := getCosignSignatureTagFromSubjectDigest(subjectDigestStr)
_, err := ref.getManifest(upstreamRepo, cosignSignatureTag)
_, _, err := ref.getManifest(upstreamRepo, cosignSignatureTag)

return err == nil
}

func (ref CosignReference) canSkipReferences(localRepo, cosignTag string, manifest *ispec.Manifest) (
func (ref CosignReference) canSkipReferences(localRepo, digest string, manifest *ispec.Manifest) (
bool, error,
) {
if manifest == nil {
return true, nil
}

imageStore := ref.storeController.GetImageStore(localRepo)
// check cosign signature already synced

var localManifest ispec.Manifest

/* we need to use tag (cosign format: sha256-$IMAGE_TAG.sig) instead of digest to get local cosign manifest
because of an issue where cosign digests differs between upstream and downstream */

localManifestBuf, _, _, err := imageStore.GetImageManifest(localRepo, cosignTag)
// check cosign signature already synced
_, localDigest, _, err := imageStore.GetImageManifest(localRepo, digest)
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) {
return false, nil
}

ref.log.Error().Str("errorType", common.TypeOf(err)).Err(err).
Str("repository", localRepo).Str("reference", cosignTag).
Str("repository", localRepo).Str("reference", digest).
Msg("couldn't get local cosign manifest")

return false, err
}

err = json.Unmarshal(localManifestBuf, &localManifest)
if err != nil {
ref.log.Error().Str("errorType", common.TypeOf(err)).
Str("repository", localRepo).Str("reference", cosignTag).
Err(err).Msg("couldn't unmarshal local cosign signature manifest")

return false, err
}

if !manifestsEqual(localManifest, *manifest) {
ref.log.Info().Str("repository", localRepo).Str("reference", cosignTag).
Msg("upstream cosign signatures changed, syncing again")

if localDigest.String() != digest {
return false, nil
}

ref.log.Info().Str("repository", localRepo).Str("reference", cosignTag).
Msg("skipping syncing cosign signature, already synced")
ref.log.Info().Str("repository", localRepo).Str("reference", digest).
Msg("skipping syncing cosign reference, already synced")

return true, nil
}

func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestStr string) error {
func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestStr string) ([]godigest.Digest, error) {
cosignTags := getCosignTagsFromSubjectDigest(subjectDigestStr)

refsDigests := make([]godigest.Digest, 0, len(cosignTags))

for _, cosignTag := range cosignTags {
manifest, err := ref.getManifest(remoteRepo, cosignTag)
if err != nil && errors.Is(err, zerr.ErrSyncReferrerNotFound) {
return err
manifest, manifestBuf, err := ref.getManifest(remoteRepo, cosignTag)
if err != nil {
if errors.Is(err, zerr.ErrSyncReferrerNotFound) {
continue
}

return refsDigests, err
}

skip, err := ref.canSkipReferences(localRepo, cosignTag, manifest)
digest := godigest.FromBytes(manifestBuf)

skip, err := ref.canSkipReferences(localRepo, digest.String(), manifest)
if err != nil {
ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("couldn't check if the remote image cosign reference can be skipped")
}

if skip {
refsDigests = append(refsDigests, digest)

continue
}

Expand All @@ -127,22 +120,13 @@ func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestSt

for _, blob := range manifest.Layers {
if err := syncBlob(ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil {
return err
return refsDigests, err
}
}

// sync config blob
if err := syncBlob(ref.client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, ref.log); err != nil {
return err
}

manifestBuf, err := json.Marshal(manifest)
if err != nil {
ref.log.Error().Str("errorType", common.TypeOf(err)).
Str("repository", localRepo).Str("subject", subjectDigestStr).
Err(err).Msg("couldn't marshal cosign reference manifest")

return err
return refsDigests, err
}

// push manifest
Expand All @@ -153,9 +137,11 @@ func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestSt
Str("repository", localRepo).Str("subject", subjectDigestStr).
Err(err).Msg("couldn't upload cosign reference manifest for image")

return err
return refsDigests, err
}

refsDigests = append(refsDigests, digest)

ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("successfully synced cosign reference for image")

Expand All @@ -166,7 +152,7 @@ func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestSt
isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, manifestBuf,
cosignTag)
if err != nil {
return fmt.Errorf("failed to check if cosign reference '%s@%s' is a signature: %w", localRepo,
return refsDigests, fmt.Errorf("failed to check if cosign reference '%s@%s' is a signature: %w", localRepo,
cosignTag, err)
}

Expand All @@ -176,45 +162,46 @@ func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestSt
SignatureDigest: referenceDigest.String(),
})
} else {
err = repodb.SetImageMetaFromInput(localRepo, cosignTag, manifest.MediaType,
err = repodb.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest,
referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo),
ref.repoDB, ref.log)
}

if err != nil {
return fmt.Errorf("failed to set metadata for cosign reference in '%s@%s': %w", localRepo, subjectDigestStr, err)
return refsDigests, fmt.Errorf("failed to set metadata for cosign reference in '%s@%s': %w",
localRepo, subjectDigestStr, err)
}

ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("repoDB: successfully added cosign reference for image")
}
}

return nil
return refsDigests, nil
}

func (ref CosignReference) getManifest(repo, cosignTag string) (*ispec.Manifest, error) {
func (ref CosignReference) getManifest(repo, cosignTag string) (*ispec.Manifest, []byte, error) {
var cosignManifest ispec.Manifest

_, _, statusCode, err := ref.client.MakeGetRequest(&cosignManifest, ispec.MediaTypeImageManifest,
body, _, statusCode, err := ref.client.MakeGetRequest(&cosignManifest, ispec.MediaTypeImageManifest,
"v2", repo, "manifests", cosignTag)
if err != nil {
if statusCode == http.StatusNotFound {
ref.log.Debug().Str("errorType", common.TypeOf(err)).
Str("repository", repo).Str("tag", cosignTag).
Err(err).Msg("couldn't find any cosign manifest for image")

return nil, zerr.ErrSyncReferrerNotFound
return nil, nil, zerr.ErrSyncReferrerNotFound
}

ref.log.Error().Str("errorType", common.TypeOf(err)).
Str("repository", repo).Str("tag", cosignTag).Int("statusCode", statusCode).
Err(err).Msg("couldn't get cosign manifest for image")

return nil, err
return nil, nil, err
}

return &cosignManifest, nil
return &cosignManifest, body, nil
}

func getCosignSignatureTagFromSubjectDigest(digestStr string) string {
Expand Down
Loading

0 comments on commit 7321edf

Please sign in to comment.