#37 update s3 image storage logic

src/compose/client.go

@@ -643,7 +643,9 @@ func (client *DockerClient) pullImageForContainers(forceUpdate bool, vars templa
 			continue
 		}
 
-		if img, err = client.Docker.InspectImage(container.Image.StringNoStorage()); err == docker.ErrNoSuchImage || forceUpdate {
+		isSha := container.Image.TagIsSha()
+
+		if img, err = client.Docker.InspectImage(container.Image.String()); err == docker.ErrNoSuchImage || (forceUpdate && !isSha) {
 			log.Infof("Pulling image: %s for %s", container.Image, container.Name)
 			if img, err = PullDockerImage(client.Docker, container.Image, client.Auth.ToDockerAPI()); err != nil {
 				err = fmt.Errorf("Failed to pull image %s for container %s, error: %s", container.Image, container.Name, err)

src/compose/container.go

@@ -219,7 +219,7 @@ func (a *Container) CreateContainerOptions() (*docker.CreateContainerOptions, er
 	labels["rocker-compose-config"] = string(yamlData)
 
 	apiConfig.Labels = labels
-	apiConfig.Image = a.Image.StringNoStorage()
+	apiConfig.Image = a.Image.String()
 
 	return &docker.CreateContainerOptions{
 		Name:       a.Name.String(),

src/compose/docker.go

@@ -19,18 +19,14 @@ package compose
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 
 	log "github.com/Sirupsen/logrus"
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/term"
 	"github.com/fsouza/go-dockerclient"
 	"github.com/grammarly/rocker/src/rocker/imagename"
+	"github.com/grammarly/rocker/src/rocker/storage/s3"
 )
 
 const emptyImageName = "gliderlabs/alpine:3.2"
@@ -97,7 +93,8 @@ func GetBridgeIP(client *docker.Client) (ip string, err error) {
 // PullDockerImage pulls an image and streams to a logger respecting terminal features
 func PullDockerImage(client *docker.Client, image *imagename.ImageName, auth *docker.AuthConfiguration) (*docker.Image, error) {
 	if image.Storage == imagename.StorageS3 {
-		return PullDockerImageS3(client, image)
+		s3storage := s3.New(client, os.TempDir())
+		return s3storage.Pull(image.String())
 	}
 
 	pipeReader, pipeWriter := io.Pipe()
@@ -145,53 +142,3 @@ func PullDockerImage(client *docker.Client, image *imagename.ImageName, auth *do
 
 	return img, nil
 }
-
-// PullDockerImageS3 imports docker image from tar artifact stored on S3
-func PullDockerImageS3(client *docker.Client, img *imagename.ImageName) (*docker.Image, error) {
-
-	// TODO: here we use tmp file, but we can stream from S3 directly to Docker
-	tmpf, err := ioutil.TempFile("", "rocker_image_")
-	if err != nil {
-		return nil, err
-	}
-	defer os.Remove(tmpf.Name())
-
-	svc := s3.New(session.New(), &aws.Config{Region: aws.String("us-east-1")})
-
-	// Create a downloader with the s3 client and custom options
-	downloader := s3manager.NewDownloaderWithClient(svc, func(d *s3manager.Downloader) {
-		d.PartSize = 64 * 1024 * 1024 // 64MB per part
-	})
-
-	downloadParams := &s3.GetObjectInput{
-		Bucket: aws.String(img.Registry),
-		Key:    aws.String(img.Name + "/" + img.Tag + ".tar"),
-	}
-
-	log.Infof("| Import s3://%s/%s.tar to %s", img.NameWithRegistry(), img.Tag, tmpf.Name())
-
-	if _, err := downloader.Download(tmpf, downloadParams); err != nil {
-		return nil, fmt.Errorf("Failed to download object from S3, error: %s", err)
-	}
-
-	fd, err := os.Open(tmpf.Name())
-	if err != nil {
-		return nil, err
-	}
-	defer fd.Close()
-
-	loadOptions := docker.LoadImageOptions{
-		InputStream: fd,
-	}
-
-	if err := client.LoadImage(loadOptions); err != nil {
-		return nil, fmt.Errorf("Failed to import image, error: %s", err)
-	}
-
-	image, err := client.InspectImage(img.StringNoStorage())
-	if err != nil {
-		return nil, fmt.Errorf("Failed to inspect image %s after pull, error: %s", img, err)
-	}
-
-	return image, nil
-}

vendor/manifest

@@ -145,13 +145,6 @@
 			"branch": "v1",
 			"path": "/src/rocker/debugtrap"
 		},
-		{
-			"importpath": "github.com/grammarly/rocker/src/rocker/imagename",
-			"repository": "https://github.com/grammarly/rocker",
-			"revision": "89d1e24bc79d1fdeb7cc848b6092ca3b9deb59f7",
-			"branch": "f-s3",
-			"path": "/src/rocker/imagename"
-		},
 		{
 			"importpath": "github.com/go-ini/ini",
 			"repository": "https://github.com/go-ini/ini",
@@ -169,6 +162,20 @@
 			"repository": "https://github.com/aws/aws-sdk-go",
 			"revision": "d4677067b535e7a06201ce491b41f4b73dcc73a9",
 			"branch": "master"
+		},
+		{
+			"importpath": "github.com/grammarly/rocker/src/rocker/storage",
+			"repository": "https://github.com/grammarly/rocker",
+			"revision": "5a8eb707986131d33a3ec12768e4bc6b29aa2f3d",
+			"branch": "f-s3",
+			"path": "/src/rocker/storage"
+		},
+		{
+			"importpath": "github.com/grammarly/rocker/src/rocker/imagename",
+			"repository": "https://github.com/grammarly/rocker",
+			"revision": "5a8eb707986131d33a3ec12768e4bc6b29aa2f3d",
+			"branch": "f-s3",
+			"path": "/src/rocker/imagename"
 		}
 	]
 }

vendor/src/github.com/grammarly/rocker/src/rocker/imagename/artifact.go

@@ -46,6 +46,18 @@ func (a *Artifact) GetFileName() string {
 	return fmt.Sprintf("%s_%s.yml", imageName, a.Name.GetTag())
 }
 
+// SetDigest sets the digest and forms the Addressable property
+func (a *Artifact) SetDigest(digest string) {
+	a.Digest = digest
+	if strings.HasPrefix(a.Digest, "sha256:") {
+		// for digest sha256:blabla
+		a.Addressable = fmt.Sprintf("%s@%s", a.Name.NameWithRegistry(), a.Digest)
+	} else {
+		// for digest sha256-blabla (tag)
+		a.Addressable = fmt.Sprintf("%s:%s", a.Name.NameWithRegistry(), a.Digest)
+	}
+}
+
 // Len returns the length of image tags
 func (a *Artifacts) Len() int {
 	return len(a.RockerArtifacts)

vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go

@@ -66,24 +66,28 @@ func New(image string, tag string) *ImageName {
 		dockerImage.SetTag(tag)
 	}
 
+	// default storage driver
+	dockerImage.Storage = StorageRegistry
+
 	// In case storage is specified, e.g. s3://bucket-name/image-name
 	storages := []string{StorageRegistry, StorageS3}
+	firstIsHost := false
 
 	for _, storage := range storages {
-		prefix := storage + "://"
+		prefix := storage + ":"
 
 		if strings.HasPrefix(image, prefix) {
-			nameParts := strings.SplitN(strings.TrimPrefix(image, prefix), "/", 2)
-			dockerImage.Registry = nameParts[0]
-			dockerImage.Name = nameParts[1]
+			image = strings.TrimPrefix(image, prefix)
 			dockerImage.Storage = storage
-			return dockerImage
+			firstIsHost = true
+			break
 		}
 	}
 
 	nameParts := strings.SplitN(image, "/", 2)
 
-	firstIsHost := strings.Contains(nameParts[0], ".") ||
+	firstIsHost = firstIsHost ||
+		strings.Contains(nameParts[0], ".") ||
 		strings.Contains(nameParts[0], ":") ||
 		nameParts[0] == "localhost"
 
@@ -94,7 +98,12 @@ func New(image string, tag string) *ImageName {
 		dockerImage.Name = nameParts[1]
 	}
 
-	dockerImage.Storage = StorageRegistry
+	// Minor validation
+	if dockerImage.Storage == StorageS3 {
+		if dockerImage.Registry == "" {
+			panic("Image with S3 storage driver requires bucket name to be specified: " + image)
+		}
+	}
 
 	return dockerImage
 }
@@ -122,17 +131,7 @@ func ParseRepositoryTag(repos string) (string, string) {
 
 // String returns the string representation of the current image name
 func (img ImageName) String() string {
-	str := img.StringNoStorage()
-	if img.Storage != StorageRegistry {
-		str = img.Storage + "://" + str
-	}
-	return str
-}
-
-// StringNoStorage returns string representation with no storage specified
-// e.g. s3://bucket-name/image-name will return bucket-name/image-name
-func (img ImageName) StringNoStorage() string {
-	if img.TagIsSha() {
+	if img.TagIsDigest() {
 		return img.NameWithRegistry() + "@" + img.GetTag()
 	}
 	return img.NameWithRegistry() + ":" + img.GetTag()
@@ -143,9 +142,16 @@ func (img ImageName) HasTag() bool {
 	return img.Tag != ""
 }
 
-// TagIsSha returns true if the tag is content addressable sha256
+// TagIsSha returns true if the tag is content addressable sha256 but can also be a tag
 // e.g. golang@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11
+// or golang:sha256-ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11
 func (img ImageName) TagIsSha() bool {
+	return strings.HasPrefix(img.Tag, "sha256:") || strings.HasPrefix(img.Tag, "sha256-")
+}
+
+// TagIsDigest returns true if the tag is content addressable sha256
+// e.g. golang@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11
+func (img ImageName) TagIsDigest() bool {
 	return strings.HasPrefix(img.Tag, "sha256:")
 }
 
@@ -229,6 +235,9 @@ func (img ImageName) NameWithRegistry() string {
 	if img.Registry != "" {
 		registryPrefix = img.Registry + "/"
 	}
+	if img.Storage != StorageRegistry {
+		registryPrefix = img.Storage + ":" + registryPrefix
+	}
 	return registryPrefix + img.Name
 }
 

vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go

@@ -494,20 +494,33 @@ func TestImagename_ToYaml(t *testing.T) {
 }
 
 func TestImagename_S3_Basic(t *testing.T) {
-	img := NewFromString("s3://bucket-name/image-name:1.2.3")
+	img := NewFromString("s3:bucket-name/image-name:1.2.3")
 	assert.Equal(t, "bucket-name", img.Registry)
 	assert.Equal(t, "image-name", img.Name)
+	assert.Equal(t, false, img.TagIsSha())
 	assert.Equal(t, "1.2.3", img.GetTag())
-	assert.Equal(t, "bucket-name/image-name", img.NameWithRegistry())
-	assert.Equal(t, "s3://bucket-name/image-name:1.2.3", img.String())
+	assert.Equal(t, "s3:bucket-name/image-name", img.NameWithRegistry())
+	assert.Equal(t, "s3:bucket-name/image-name:1.2.3", img.String())
 }
 
-func TestImagename_S3_Etag(t *testing.T) {
-	img := NewFromString("s3://bucket-name/image-name@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11")
+func TestImagename_S3_Digest(t *testing.T) {
+	img := NewFromString("s3:bucket-name/image-name@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11")
 	assert.Equal(t, "bucket-name", img.Registry)
 	assert.Equal(t, "image-name", img.Name)
 	assert.Equal(t, true, img.TagIsSha())
-	assert.Equal(t, "bucket-name/image-name", img.NameWithRegistry())
+	assert.Equal(t, true, img.TagIsDigest())
+	assert.Equal(t, "s3:bucket-name/image-name", img.NameWithRegistry())
 	assert.Equal(t, "sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11", img.GetTag())
-	assert.Equal(t, "s3://bucket-name/image-name@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11", img.String())
+	assert.Equal(t, "s3:bucket-name/image-name@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11", img.String())
+}
+
+func TestImagename_S3_Sha(t *testing.T) {
+	img := NewFromString("s3:bucket-name/image-name:sha256-ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11")
+	assert.Equal(t, "bucket-name", img.Registry)
+	assert.Equal(t, "image-name", img.Name)
+	assert.Equal(t, true, img.TagIsSha())
+	assert.Equal(t, false, img.TagIsDigest())
+	assert.Equal(t, "s3:bucket-name/image-name", img.NameWithRegistry())
+	assert.Equal(t, "sha256-ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11", img.GetTag())
+	assert.Equal(t, "s3:bucket-name/image-name:sha256-ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11", img.String())
 }

vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go

@@ -122,7 +122,7 @@ func s3ListTags(image *ImageName) (images []*ImageName, err error) {
 		}
 
 		imgName := strings.Join(split[:len(split)-1], "/")
-		imgName = fmt.Sprintf("s3://%s/%s", image.Registry, imgName)
+		imgName = fmt.Sprintf("s3:%s/%s", image.Registry, imgName)
 
 		tag := strings.TrimSuffix(split[len(split)-1], ".tar")
 		candidate := New(imgName, tag)