Support OCI 1.0

SOCI relies on being able to associate an index with an image. This
capability was introduced in OCI distribution and image specs version
1.1 as Artifacts and the Referrers API respectively.

In order to drive adoption, an OCI 1.0 compatible fallback was added
that encodes artifacts as Image Manifests and stores the Referrers API
content in an Image Index.

This PR adds the ability to serialize and deserialize SOCI indices as
either OCI Artifacts or OCI Images. There is no automatic OCI 1.0
fallback when creating images. The user of the CLI/library must
explicitly request an OCI 1.0 compatible Image Manifest. The snapshotter
will be able to automatically load a SOCI index in either serialized
form.

Signed-off-by: Kern Walster <walster@amazon.com>

cmd/soci/commands/create.go

@@ -45,6 +45,7 @@ var CreateCommand = cli.Command{
 	ArgsUsage: "[flags] <image_ref>",
 	Flags: append(
 		internal.PlatformFlags,
+		internal.LegacyRegistryFlag,
 		cli.Int64Flag{
 			Name:  spanSizeFlag,
 			Usage: "Span size that soci index uses to segment layer data. Default is 4 MiB",
@@ -96,12 +97,16 @@ var CreateCommand = cli.Command{
 			return err
 		}
 
+		builderOpts := []soci.BuildOption{
+			soci.WithMinLayerSize(minLayerSize),
+			soci.WithSpanSize(spanSize),
+			soci.WithBuildToolIdentifier(buildToolIdentifier),
+		}
+		if cliContext.Bool(internal.LegacyRegistryFlagName) {
+			builderOpts = append(builderOpts, soci.WithLegacyRegistrySupport)
+		}
 		for _, plat := range ps {
-			builder, err := soci.NewIndexBuilder(cs, blobStore,
-				soci.WithMinLayerSize(minLayerSize),
-				soci.WithSpanSize(spanSize),
-				soci.WithBuildToolIdentifier(buildToolIdentifier),
-				soci.WithPlatform(plat))
+			builder, err := soci.NewIndexBuilder(cs, blobStore, append(builderOpts, soci.WithPlatform(plat))...)
 
 			if err != nil {
 				return err

cmd/soci/commands/index/list.go

@@ -155,7 +155,7 @@ var listCommand = cli.Command{
 		}
 
 		writer := tabwriter.NewWriter(os.Stdout, 8, 8, 4, ' ', 0)
-		writer.Write([]byte("DIGEST\tSIZE\tIMAGE REF\tPLATFORM\tCREATED\t\n"))
+		writer.Write([]byte("DIGEST\tSIZE\tIMAGE REF\tPLATFORM\tMEDIA TYPE\tCREATED\t\n"))
 
 		for _, ae := range artifacts {
 			imgs, _ := is.List(ctx, fmt.Sprintf("target.digest==%s", ae.ImageDigest))
@@ -174,11 +174,12 @@ var listCommand = cli.Command{
 
 func writeArtifactEntry(w io.Writer, ae *soci.ArtifactEntry, imageRef string) {
 	w.Write([]byte(fmt.Sprintf(
-		"%s\t%d\t%s\t%s\t%s\t\n",
+		"%s\t%d\t%s\t%s\t%s\t%s\t\n",
 		ae.Digest,
 		ae.Size,
 		imageRef,
 		ae.Platform,
+		ae.MediaType,
 		getDuration(ae.CreatedAt),
 	)))
 }

cmd/soci/commands/internal/spec.go

@@ -0,0 +1,31 @@
+/*
+   Copyright The Soci Snapshotter Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package internal
+
+import "github.com/urfave/cli"
+
+const (
+	LegacyRegistryFlagName = "legacy-registry"
+)
+
+var LegacyRegistryFlag = cli.BoolFlag{
+	Name: LegacyRegistryFlagName,
+	Usage: `Whether to create the SOCI index for a legacy registry. OCI 1.1 added support for associating artifacts such as soci indices with images.
+     There is a mechanism to emulate this behavior with OCI 1.0 registries by pretending that the SOCI index
+     is itself an image. This option should only be use if the SOCI index will be pushed to a
+     registry which does not support OCI 1.1 features.`,
+}

fs/artifact_fetcher.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
-	"encoding/json"
 	"fmt"
 	"io"
 
@@ -183,13 +182,14 @@ func FetchSociArtifacts(ctx context.Context, refspec reference.Spec, indexDesc o
 	cw := new(ioutils.CountWriter)
 	tee := io.TeeReader(indexReader, cw)
 
-	index, err := soci.NewIndexFromReader(tee)
+	var index soci.Index
+	err = soci.DecodeIndex(tee, &index)
 	if err != nil {
 		return nil, fmt.Errorf("cannot deserialize byte data to index: %w", err)
 	}
 
 	if !local {
-		b, err := json.Marshal(index)
+		b, err := soci.MarshalIndex(&index)
 		if err != nil {
 			return nil, err
 		}
@@ -223,5 +223,5 @@ func FetchSociArtifacts(ctx context.Context, refspec reference.Spec, indexDesc o
 		return nil, err
 	}
 
-	return index, nil
+	return &index, nil
 }

integration/create_test.go

@@ -66,7 +66,8 @@ func TestSociCreateSparseIndex(t *testing.T) {
 			indexDigest := buildIndex(sh, imgInfo, withMinLayerSize(tt.minLayerSize))
 			checkpoints := fetchContentFromPath(sh, blobStorePath+"/"+trimSha256Prefix(indexDigest))
 
-			index, err := soci.NewIndexFromReader(bytes.NewReader(checkpoints))
+			var index soci.Index
+			err := soci.DecodeIndex(bytes.NewReader(checkpoints), &index)
 			if err != nil {
 				t.Fatalf("cannot get index data: %v", err)
 			}
@@ -84,7 +85,7 @@ func TestSociCreateSparseIndex(t *testing.T) {
 				}
 			}
 
-			validateSociIndex(t, sh, *index, manifestDigest, includedLayers)
+			validateSociIndex(t, sh, index, manifestDigest, includedLayers)
 		})
 	}
 }
@@ -145,7 +146,8 @@ func TestSociCreate(t *testing.T) {
 			imgInfo := dockerhub(tt.containerImage, withPlatform(platform))
 			indexDigest := buildIndex(sh, imgInfo, withMinLayerSize(0))
 			checkpoints := fetchContentFromPath(sh, blobStorePath+"/"+trimSha256Prefix(indexDigest))
-			sociIndex, err := soci.NewIndexFromReader(bytes.NewReader(checkpoints))
+			var sociIndex soci.Index
+			err := soci.DecodeIndex(bytes.NewReader(checkpoints), &sociIndex)
 			if err != nil {
 				t.Fatalf("cannot get soci index: %v", err)
 			}
@@ -155,7 +157,7 @@ func TestSociCreate(t *testing.T) {
 				t.Fatalf("failed to get manifest digest: %v", err)
 			}
 
-			validateSociIndex(t, sh, *sociIndex, m, nil)
+			validateSociIndex(t, sh, sociIndex, m, nil)
 		})
 	}
 }

integration/index_test.go

@@ -19,7 +19,6 @@ package integration
 import (
 	"bufio"
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"strings"
 	"testing"
@@ -280,7 +279,7 @@ func sociIndexFromDigest(sh *shell.Shell, indexDigest string) (index soci.Index,
 	if err != nil {
 		return
 	}
-	if err = json.Unmarshal(rawSociIndexJSON, &index); err != nil {
+	if err = soci.UnmarshalIndex(rawSociIndexJSON, &index); err != nil {
 		err = fmt.Errorf("invalid soci index from digest %s: %s", indexDigest, err)
 	}
 	return

integration/pull_test.go

@@ -828,7 +828,7 @@ func TestRpullImageThenRemove(t *testing.T) {
 
 	rawJSON := sh.O("soci", "index", "info", indexDigest)
 	var sociIndex soci.Index
-	if err := json.Unmarshal(rawJSON, &sociIndex); err != nil {
+	if err := soci.UnmarshalIndex(rawJSON, &sociIndex); err != nil {
 		t.Fatalf("invalid soci index from digest %s: %v", indexDigest, rawJSON)
 	}
 

integration/push_test.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/awslabs/soci-snapshotter/soci"
 	"github.com/awslabs/soci-snapshotter/util/testutil"
 	"github.com/containerd/containerd/platforms"
 )
@@ -134,3 +135,83 @@ func TestPushAlwaysMostRecentlyCreatedIndex(t *testing.T) {
 		})
 	}
 }
+
+func TestLegacyOCI(t *testing.T) {
+	tests := []struct {
+		name                  string
+		registryImage         string
+		supportLegacyRegistry bool
+		expectError           bool
+	}{
+		{
+			name:                  "OCI 1.1 Artifacts succeed with OCI 1.1 registry",
+			registryImage:         oci11RegistryImage,
+			supportLegacyRegistry: false,
+			expectError:           false,
+		},
+		{
+			name:                  "OCI 1.0 Artifacts succeed with OCI 1.1 registry",
+			registryImage:         oci11RegistryImage,
+			supportLegacyRegistry: true,
+			expectError:           false,
+		},
+		{
+			name:                  "OCI 1.1 Artifacts fail with OCI 1.0 registry",
+			registryImage:         oci10RegistryImage,
+			supportLegacyRegistry: false,
+			expectError:           true,
+		},
+		{
+			name:                  "OCI 1.0 Artifacts succeed with OCI 1.0 registry",
+			registryImage:         oci10RegistryImage,
+			supportLegacyRegistry: true,
+			expectError:           false,
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			regConfig := newRegistryConfig()
+			sh, done := newShellWithRegistry(t, regConfig, withRegistryImageRef(tc.registryImage))
+			defer done()
+
+			if err := testutil.WriteFileContents(sh, defaultContainerdConfigPath, getContainerdConfigYaml(t, false), 0600); err != nil {
+				t.Fatalf("failed to write %v: %v", defaultContainerdConfigPath, err)
+			}
+			if err := testutil.WriteFileContents(sh, defaultSnapshotterConfigPath, getSnapshotterConfigYaml(t, false), 0600); err != nil {
+				t.Fatalf("failed to write %v: %v", defaultSnapshotterConfigPath, err)
+			}
+			rebootContainerd(t, sh, "", "")
+
+			imageName := ubuntuImage
+			copyImage(sh, dockerhub(imageName), regConfig.mirror(imageName))
+			var buildOpts []indexBuildOption
+			if tc.supportLegacyRegistry {
+				buildOpts = append(buildOpts, withLegacyRegistrySupport)
+			}
+
+			indexDigest := buildIndex(sh, regConfig.mirror(imageName), buildOpts...)
+			rawJSON := sh.O("soci", "index", "info", indexDigest)
+			var sociIndex soci.Index
+			if err := soci.UnmarshalIndex(rawJSON, &sociIndex); err != nil {
+				t.Fatalf("invalid soci index from digest %s: %v", indexDigest, rawJSON)
+			}
+			_, err := sh.OLog("soci", "push", "--user", regConfig.creds(), regConfig.mirror(imageName).ref)
+			hasError := err != nil
+			if hasError != tc.expectError {
+				t.Fatalf("unexpected error state: expected error? %v, got %v", tc.expectError, err)
+			} else if hasError {
+				// if we have an error and we expected an error, the test is done
+				return
+			}
+			sh.X("rm", "-rf", "/var/lib/soci-snapshotter-grpc/content/blobs/sha256")
+
+			sh.X("soci", "image", "rpull", "--user", regConfig.creds(), "--soci-index-digest", indexDigest, regConfig.mirror(imageName).ref)
+			if err := sh.Err(); err != nil {
+				t.Fatalf("failed to rpull: %v", err)
+			}
+			checkFuseMounts(t, sh, len(sociIndex.Blobs))
+		})
+	}
+}

integration/testutil.go

@@ -68,8 +68,13 @@ const (
 	dockerLibrary                = "public.ecr.aws/docker/library/"
 	blobStorePath                = "/var/lib/soci-snapshotter-grpc/content/blobs/sha256"
 	containerdBlobStorePath      = "/var/lib/containerd/io.containerd.content.v1.content/blobs/sha256"
+	// Registry images to use in the test infrastructure. These are not intended to be used
+	// as images in the test itself, but just when we're setting up docker compose.
+	oci11RegistryImage = "ghcr.io/oci-playground/registry:v3.0.0-alpha.1"
+	oci10RegistryImage = "docker.io/library/registry:2"
 )
 
+// These are images that we use in our integration tests
 const (
 	alpineImage   = "alpine:latest"
 	nginxImage    = "nginx:latest"
@@ -272,13 +277,27 @@ func (c registryConfig) mirror(imageName string, opts ...imageOpt) imageInfo {
 }
 
 type registryOptions struct {
-	network string
+	network          string
+	registryImageRef string
+}
+
+func defaultRegistryOptions() registryOptions {
+	return registryOptions{
+		network:          "",
+		registryImageRef: oci11RegistryImage,
+	}
 }
 
 type registryOpt func(o *registryOptions)
 
+func withRegistryImageRef(ref string) registryOpt {
+	return func(o *registryOptions) {
+		o.registryImageRef = ref
+	}
+}
+
 func newShellWithRegistry(t *testing.T, r registryConfig, opts ...registryOpt) (sh *shell.Shell, done func() error) {
-	var rOpts registryOptions
+	rOpts := defaultRegistryOptions()
 	for _, o := range opts {
 		o(&rOpts)
 	}
@@ -356,7 +375,7 @@ services:
     volumes:
     - /dev/fuse:/dev/fuse
   registry:
-    image: ghcr.io/oci-playground/registry:v3.0.0-alpha.1
+    image: {{.RegistryImageRef}}
     container_name: {{.RegistryHost}}
     environment:
     - REGISTRY_AUTH=htpasswd
@@ -369,19 +388,21 @@ services:
     - {{.AuthDir}}:/auth:ro
 {{.NetworkConfig}}
 `, struct {
-		ServiceName     string
-		ImageContextDir string
-		TargetStage     string
-		RegistryHost    string
-		AuthDir         string
-		NetworkConfig   string
+		ServiceName      string
+		ImageContextDir  string
+		TargetStage      string
+		RegistryImageRef string
+		RegistryHost     string
+		AuthDir          string
+		NetworkConfig    string
 	}{
-		ServiceName:     serviceName,
-		ImageContextDir: pRoot,
-		TargetStage:     targetStage,
-		RegistryHost:    r.host,
-		AuthDir:         authDir,
-		NetworkConfig:   networkConfig,
+		ServiceName:      serviceName,
+		ImageContextDir:  pRoot,
+		TargetStage:      targetStage,
+		RegistryImageRef: rOpts.registryImageRef,
+		RegistryHost:     r.host,
+		AuthDir:          authDir,
+		NetworkConfig:    networkConfig,
 	}), cOpts...)
 	if err != nil {
 		t.Fatalf("failed to prepare compose: %v", err)