Add --fast flag to crane validate (#1013)

* Add --fast flag to crane validate

Add validate.Fast option
Add Exists to remote layers
Add Exists to layout
Add Exists to partial

Simplify some of our typechecking logic with unwrap().

Stop double-wrapping layoutImage layers.

* Return nil errors for expected non-existence

* Document partial.Exists

cmd/crane/cmd/validate.go

@@ -26,7 +26,10 @@ import (
 
 // NewCmdValidate creates a new cobra.Command for the validate subcommand.
 func NewCmdValidate(options *[]crane.Option) *cobra.Command {
-	var tarballPath, remoteRef string
+	var (
+		tarballPath, remoteRef string
+		fast                   bool
+	)
 
 	validateCmd := &cobra.Command{
 		Use:   "validate",
@@ -45,7 +48,11 @@ func NewCmdValidate(options *[]crane.Option) *cobra.Command {
 					return fmt.Errorf("failed to read image %s: %v", flag, err)
 				}
 
-				if err := validate.Image(img); err != nil {
+				opt := []validate.Option{}
+				if fast {
+					opt = append(opt, validate.Fast)
+				}
+				if err := validate.Image(img, opt...); err != nil {
 					fmt.Printf("FAIL: %s: %v\n", flag, err)
 					return err
 				} else {
@@ -57,6 +64,7 @@ func NewCmdValidate(options *[]crane.Option) *cobra.Command {
 	}
 	validateCmd.Flags().StringVar(&tarballPath, "tarball", "", "Path to tarball to validate")
 	validateCmd.Flags().StringVar(&remoteRef, "remote", "", "Name of remote image to validate")
+	validateCmd.Flags().BoolVar(&fast, "fast", false, "Skip downloading/digesting layers")
 
 	return validateCmd
 }

pkg/v1/layout/image.go

@@ -17,6 +17,7 @@ package layout
 import (
 	"fmt"
 	"io"
+	"os"
 	"sync"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -84,20 +85,20 @@ func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error)
 	}
 
 	if h == manifest.Config.Digest {
-		return partial.CompressedLayer(&compressedBlob{
+		return &compressedBlob{
 			path: li.path,
 			desc: manifest.Config,
-		}), nil
+		}, nil
 	}
 
 	for _, desc := range manifest.Layers {
 		if h == desc.Digest {
 			switch desc.MediaType {
 			case types.OCILayer, types.DockerLayer:
-				return partial.CompressedToLayer(&compressedBlob{
+				return &compressedBlob{
 					path: li.path,
 					desc: desc,
-				})
+				}, nil
 			default:
 				// TODO: We assume everything is a compressed blob, but that might not be true.
 				// TODO: Handle foreign layers.
@@ -129,3 +130,12 @@ func (b *compressedBlob) Size() (int64, error) {
 func (b *compressedBlob) MediaType() (types.MediaType, error) {
 	return b.desc.MediaType, nil
 }
+
+// See partial.Exists.
+func (b *compressedBlob) Exists() (bool, error) {
+	_, err := os.Stat(b.path.blobPath(b.desc.Digest))
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return err == nil, err
+}

pkg/v1/layout/image_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/partial"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 	"github.com/google/go-containerregistry/pkg/v1/validate"
 )
@@ -99,6 +100,12 @@ func TestImage(t *testing.T) {
 	if got, want := mediaType, types.DockerLayer; got != want {
 		t.Fatalf("MediaType(); want: %q got: %q", want, got)
 	}
+
+	if ok, err := partial.Exists(layers[0]); err != nil {
+		t.Fatal(err)
+	} else if got, want := ok, true; got != want {
+		t.Errorf("Exists() = %t != %t", got, want)
+	}
 }
 
 func TestImageWithEmptyHash(t *testing.T) {

pkg/v1/partial/README.md

@@ -64,3 +64,19 @@ there are cases where it is very helpful to know the layer size, e.g. when
 writing the uncompressed layer into a tarball.
 
 See [`#655`](https://github.com/google/go-containerregistry/pull/655).
+
+### [`partial.Exists`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#Exists)
+
+We generally don't care about the existence of something as granular as a
+layer, and would rather ensure all the invariants of an image are upheld via
+the `validate` package. However, there are situations where we want to do a
+quick smoke test to ensure that the underlying storage engine hasn't been
+corrupted by something e.g. deleting files or blobs. Thus, we've exposed an
+optional `Exists` method that does an existence check without actually reading
+any bytes.
+
+The `remote` package implements this via `HEAD` requests.
+
+The `layout` package implements this via `os.Stat`.
+
+See [`#838`](https://github.com/google/go-containerregistry/pull/838).

pkg/v1/partial/compressed_test.go

@@ -79,6 +79,14 @@ func TestRemote(t *testing.T) {
 	if diff := cmp.Diff(d, m.Layers[0].Digest); diff != "" {
 		t.Errorf("mismatched digest: %v", diff)
 	}
+
+	ok, err := partial.Exists(layer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := ok, true; got != want {
+		t.Errorf("Exists() = %t != %t", got, want)
+	}
 }
 
 type noDiffID struct {

pkg/v1/partial/uncompressed_test.go

@@ -214,4 +214,21 @@ func TestUncompressed(t *testing.T) {
 	if _, err := partial.Descriptor(img); err != nil {
 		t.Fatalf("partial.Descriptor: %v", err)
 	}
+
+	layers, err := img.Layers()
+	if err != nil {
+		t.Fatal(err)
+	}
+	layer, err := partial.UncompressedToLayer(&fastpathLayer{layers[0]})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ok, err := partial.Exists(layer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := ok, true; got != want {
+		t.Errorf("Exists() = %t != %t", got, want)
+	}
 }

pkg/v1/partial/with.go

@@ -297,33 +297,10 @@ type Describable interface {
 // UncompressedToImage.
 func Descriptor(d Describable) (*v1.Descriptor, error) {
 	// If Describable implements Descriptor itself, return that.
-	if wd, ok := d.(withDescriptor); ok {
+	if wd, ok := unwrap(d).(withDescriptor); ok {
 		return wd.Descriptor()
 	}
 
-	// Otherwise, try to unwrap any partial implementations to see
-	// if the wrapped struct implements Descriptor.
-	if ule, ok := d.(*uncompressedLayerExtender); ok {
-		if wd, ok := ule.UncompressedLayer.(withDescriptor); ok {
-			return wd.Descriptor()
-		}
-	}
-	if cle, ok := d.(*compressedLayerExtender); ok {
-		if wd, ok := cle.CompressedLayer.(withDescriptor); ok {
-			return wd.Descriptor()
-		}
-	}
-	if uie, ok := d.(*uncompressedImageExtender); ok {
-		if wd, ok := uie.UncompressedImageCore.(withDescriptor); ok {
-			return wd.Descriptor()
-		}
-	}
-	if cie, ok := d.(*compressedImageExtender); ok {
-		if wd, ok := cie.CompressedImageCore.(withDescriptor); ok {
-			return wd.Descriptor()
-		}
-	}
-
 	// If all else fails, compute the descriptor from the individual methods.
 	var (
 		desc v1.Descriptor
@@ -354,23 +331,10 @@ type withUncompressedSize interface {
 // for streaming layers.
 func UncompressedSize(l v1.Layer) (int64, error) {
 	// If the layer implements UncompressedSize itself, return that.
-	if wus, ok := l.(withUncompressedSize); ok {
+	if wus, ok := unwrap(l).(withUncompressedSize); ok {
 		return wus.UncompressedSize()
 	}
 
-	// Otherwise, try to unwrap any partial implementations to see
-	// if the wrapped struct implements UncompressedSize.
-	if ule, ok := l.(*uncompressedLayerExtender); ok {
-		if wus, ok := ule.UncompressedLayer.(withUncompressedSize); ok {
-			return wus.UncompressedSize()
-		}
-	}
-	if cle, ok := l.(*compressedLayerExtender); ok {
-		if wus, ok := cle.CompressedLayer.(withUncompressedSize); ok {
-			return wus.UncompressedSize()
-		}
-	}
-
 	// The layer doesn't implement UncompressedSize, we need to compute it.
 	rc, err := l.Uncompressed()
 	if err != nil {
@@ -380,3 +344,46 @@ func UncompressedSize(l v1.Layer) (int64, error) {
 
 	return io.Copy(ioutil.Discard, rc)
 }
+
+type withExists interface {
+	Exists() (bool, error)
+}
+
+// Exists checks to see if a layer exists. This is a hack to work around the
+// mistakes of the partial package. Don't use this.
+func Exists(l v1.Layer) (bool, error) {
+	// If the layer implements Exists itself, return that.
+	if we, ok := unwrap(l).(withExists); ok {
+		return we.Exists()
+	}
+
+	// The layer doesn't implement Exists, so we hope that calling Compressed()
+	// is enough to trigger an error if the layer does not exist.
+	rc, err := l.Compressed()
+	if err != nil {
+		return false, err
+	}
+	defer rc.Close()
+
+	// We may want to try actually reading a single byte, but if we need to do
+	// that, we should just fix this hack.
+	return true, nil
+}
+
+// Recursively unwrap our wrappers so that we can check for the original implementation.
+// We might want to expose this?
+func unwrap(i interface{}) interface{} {
+	if ule, ok := i.(*uncompressedLayerExtender); ok {
+		return unwrap(ule.UncompressedLayer)
+	}
+	if cle, ok := i.(*compressedLayerExtender); ok {
+		return unwrap(cle.CompressedLayer)
+	}
+	if uie, ok := i.(*uncompressedImageExtender); ok {
+		return unwrap(uie.UncompressedImageCore)
+	}
+	if cie, ok := i.(*compressedImageExtender); ok {
+		return unwrap(cie.CompressedImageCore)
+	}
+	return i
+}

pkg/v1/partial/with_test.go

@@ -203,6 +203,10 @@ func (l *fastpathLayer) UncompressedSize() (int64, error) {
 	return 100, nil
 }
 
+func (l *fastpathLayer) Exists() (bool, error) {
+	return true, nil
+}
+
 func TestUncompressedSize(t *testing.T) {
 	randLayer, err := random.Layer(1024, types.DockerLayer)
 	if err != nil {
@@ -217,3 +221,26 @@ func TestUncompressedSize(t *testing.T) {
 		t.Errorf("UncompressedSize() = %d != %d", got, want)
 	}
 }
+
+func TestExists(t *testing.T) {
+	randLayer, err := random.Layer(1024, types.DockerLayer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fpl := &fastpathLayer{randLayer}
+	ok, err := partial.Exists(fpl)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := ok, true; got != want {
+		t.Errorf("Exists() = %t != %t", got, want)
+	}
+
+	ok, err = partial.Exists(randLayer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := ok, true; got != want {
+		t.Errorf("Exists() = %t != %t", got, want)
+	}
+}

pkg/v1/remote/descriptor.go

@@ -402,3 +402,23 @@ func (f *fetcher) headBlob(h v1.Hash) (*http.Response, error) {
 
 	return resp, nil
 }
+
+func (f *fetcher) blobExists(h v1.Hash) (bool, error) {
+	u := f.url("blobs", h.String())
+	req, err := http.NewRequest(http.MethodHead, u.String(), nil)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := f.Client.Do(req.WithContext(f.context))
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound); err != nil {
+		return false, err
+	}
+
+	return resp.StatusCode == http.StatusOK, nil
+}

pkg/v1/remote/image.go

@@ -221,6 +221,11 @@ func (rl *remoteImageLayer) Descriptor() (*v1.Descriptor, error) {
 	return partial.BlobDescriptor(rl, rl.digest)
 }
 
+// See partial.Exists.
+func (rl *remoteImageLayer) Exists() (bool, error) {
+	return rl.ri.blobExists(rl.digest)
+}
+
 // LayerByDigest implements partial.CompressedLayer
 func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) {
 	return &remoteImageLayer{

pkg/v1/remote/layer.go

@@ -43,6 +43,7 @@ func (rl *remoteLayer) Size() (int64, error) {
 	if err != nil {
 		return -1, err
 	}
+	defer resp.Body.Close()
 	return resp.ContentLength, nil
 }
 
@@ -56,6 +57,11 @@ func (rl *remoteLayer) MediaType() (types.MediaType, error) {
 	return types.DockerLayer, nil
 }
 
+// See partial.Exists.
+func (rl *remoteLayer) Exists() (bool, error) {
+	return rl.blobExists(rl.digest)
+}
+
 // Layer reads the given blob reference from a registry as a Layer. A blob
 // reference here is just a punned name.Digest where the digest portion is the
 // digest of the blob to be read and the repository portion is the repo where

pkg/v1/remote/layer_test.go

@@ -78,6 +78,12 @@ func TestRemoteLayer(t *testing.T) {
 	if err := validate.Layer(got); err != nil {
 		t.Errorf("validate.Layer: %v", err)
 	}
+
+	if ok, err := partial.Exists(got); err != nil {
+		t.Fatal(err)
+	} else if got, want := ok, true; got != want {
+		t.Errorf("Exists() = %t != %t", got, want)
+	}
 }
 
 func TestRemoteLayerDescriptor(t *testing.T) {
@@ -134,4 +140,9 @@ func TestRemoteLayerDescriptor(t *testing.T) {
 	if got, want := desc.URLs[0], "example.com"; got != want {
 		t.Errorf("layer[0].urls[0] = %s != %s", got, want)
 	}
+	if ok, err := partial.Exists(layers[0]); err != nil {
+		t.Fatal(err)
+	} else if got, want := ok, true; got != want {
+		t.Errorf("Exists() = %t != %t", got, want)
+	}
 }