add diff cmd

bin/makisu/cmd/diff.go

@@ -0,0 +1,104 @@
+package cmd
+
+import (
+	"archive/tar"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/andres-erbsen/clock"
+	"github.com/spf13/cobra"
+	"github.com/uber/makisu/lib/docker/image"
+	"github.com/uber/makisu/lib/log"
+	"github.com/uber/makisu/lib/registry"
+	"github.com/uber/makisu/lib/snapshot"
+	"github.com/uber/makisu/lib/storage"
+	"github.com/uber/makisu/lib/tario"
+)
+
+type diffCmd struct {
+	*cobra.Command
+	ignoreModTime bool
+}
+
+func getDiffCmd() *diffCmd {
+	diffCmd := &diffCmd{
+		Command: &cobra.Command{
+			Use:                   "diff <image name> <image name>",
+			DisableFlagsInUseLine: true,
+			Short:                 "Compare docker images from registry",
+		},
+	}
+
+	diffCmd.Args = func(cmd *cobra.Command, args []string) error {
+		if len(args) != 2 {
+			return errors.New("Requires two image names as arguments")
+		}
+		return nil
+	}
+
+	diffCmd.Run = func(cmd *cobra.Command, args []string) {
+		if err := diffCmd.Diff(args); err != nil {
+			log.Error(err)
+			os.Exit(1)
+		}
+	}
+
+	diffCmd.PersistentFlags().BoolVar(&diffCmd.ignoreModTime, "ignoreModTime", true, "Ignore mod time of image files when comparing images")
+	return diffCmd
+}
+
+func (cmd *diffCmd) Diff(imagesFullName []string) error {
+	log.Infof("ingore time? :%t", cmd.ignoreModTime)
+	var pullImages []image.Name
+	for _, imageFullName := range imagesFullName {
+		pullImage, err := image.ParseNameForPull(imageFullName)
+		if err != nil {
+			return fmt.Errorf("parse image %s: %s", pullImage, err)
+		}
+		pullImages = append(pullImages, pullImage)
+	}
+
+	if err := initRegistryConfig(""); err != nil {
+		return fmt.Errorf("failed to initialize registry configuration: %s", err)
+	}
+
+	store, err := storage.NewImageStore("/tmp/makisu-storage/")
+	if err != nil {
+		panic(err)
+	}
+
+	var memFSArr []*snapshot.MemFS
+	for i, pullImage := range pullImages {
+		client := registry.New(store, pullImage.GetRegistry(), pullImage.GetRepository())
+		manifest, err := client.Pull(pullImage.GetTag())
+		if err != nil {
+			panic(err)
+		}
+
+		memfs, err := snapshot.NewMemFS(clock.New(), "/tmp/makisu-storage/", nil)
+		if err != nil {
+			panic(err)
+		}
+
+		for _, descriptor := range manifest.Layers {
+			reader, err := store.Layers.GetStoreFileReader(descriptor.Digest.Hex())
+			if err != nil {
+				panic(fmt.Errorf("get reader from image %d layer: %s", i+1, err))
+			}
+			gzipReader, err := tario.NewGzipReader(reader)
+			if err != nil {
+				panic(fmt.Errorf("create gzip reader for layer: %s", err))
+			}
+			if err = memfs.UpdateFromTarReader(tar.NewReader(gzipReader), false); err != nil {
+				panic(fmt.Errorf("untar image %d layer reader: %s", i+1, err))
+			}
+		}
+		memFSArr = append(memFSArr, memfs)
+	}
+
+	log.Infof("* Diff two images")
+
+	snapshot.CompareFS(memFSArr[0], memFSArr[1], pullImages[0], pullImages[1], cmd.ignoreModTime)
+	return nil
+}

bin/makisu/cmd/root.go

@@ -76,6 +76,7 @@ func Execute() {
 	rootCmd.AddCommand(getVersionCmd())
 	rootCmd.AddCommand(getPullCmd().Command)
 	rootCmd.AddCommand(getPushCmd().Command)
+	rootCmd.AddCommand(getDiffCmd().Command)
 	if err := rootCmd.Execute(); err != nil {
 		log.Error(err)
 		os.Exit(1)

lib/snapshot/mem_fs.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/andres-erbsen/clock"
+	"github.com/uber/makisu/lib/docker/image"
 	"github.com/uber/makisu/lib/fileio"
 	"github.com/uber/makisu/lib/log"
 	"github.com/uber/makisu/lib/mountutils"
@@ -489,7 +490,7 @@ func (fs *MemFS) isUpdated(p string, hdr *tar.Header) (bool, *memFSNode, error)
 		}
 	}
 
-	similar, err := tario.IsSimilarHeader(curr.hdr, hdr)
+	similar, err := tario.IsSimilarHeader(curr.hdr, hdr, false)
 	if err != nil {
 		return false, nil, fmt.Errorf("compare header %s: %s", p, err)
 	}
@@ -599,7 +600,7 @@ func (fs *MemFS) untarOneItem(path string, header *tar.Header, r *tar.Reader) er
 		}
 
 		// If the file is already on disk, nothing needs to be done.
-		if similar, err := tario.IsSimilarHeader(localHeader, header); err != nil {
+		if similar, err := tario.IsSimilarHeader(localHeader, header, false); err != nil {
 			return fmt.Errorf("compare headers %s: %s", path, err)
 		} else if similar {
 			return nil
@@ -710,3 +711,79 @@ func (fs *MemFS) untarFile(path string, header *tar.Header, r *tar.Reader) error
 	}
 	return nil
 }
+
+// Public api for diff image purpose.
+func CompareFS(fs1, fs2 *MemFS, image1Name, image2Name image.Name, ignoreModTime bool) {
+	missing1 := make(map[string]*memFSNode)
+	missing2 := make(map[string]*memFSNode)
+	diff1 := make(map[string]*memFSNode)
+	diff2 := make(map[string]*memFSNode)
+
+	compareFS(fs1, fs2, image1Name, image2Name, missing1, missing2, diff1, diff2, ignoreModTime)
+}
+
+// compare tar files of memFSNodes.
+func compareNode(node1, node2 *memFSNode, missing1, missing2, diff1, diff2 map[string]*memFSNode, path string, ignoreModTime bool) {
+	if isSimilar, _ := tario.IsSimilarHeader(node1.hdr, node2.hdr, ignoreModTime); !isSimilar {
+		diff1[path] = node1
+		diff2[path] = node2
+	}
+
+	allChildren := make(map[string]bool)
+	for child := range node1.children {
+		allChildren[child] = true
+	}
+
+	for child := range node2.children {
+		allChildren[child] = true
+	}
+
+	for child := range allChildren {
+		nextNode1, ok1 := node1.children[child]
+		nextNode2, ok2 := node2.children[child]
+		updatedPath := filepath.Join(path, child)
+		if ok1 && ok2 {
+			compareNode(nextNode1, nextNode2, missing1, missing2, diff1, diff2, updatedPath, ignoreModTime)
+			continue
+		} else if ok1 {
+			missing2[updatedPath] = node1.children[child]
+		} else if ok2 {
+			missing1[updatedPath] = node2.children[child]
+		}
+	}
+}
+
+// log the info of a memFSNode.
+func logMemFSNodeInfo(node *memFSNode) {
+	hdr := node.hdr
+	log.Infof("%s %s %d %d %d", hdr.Name, hdr.FileInfo().Mode(), hdr.Uid, hdr.Gid, hdr.Size)
+	for _, nxtNode := range node.children {
+		logMemFSNodeInfo(nxtNode)
+	}
+}
+
+func compareFS(fs1, fs2 *MemFS, image1Name, image2Name image.Name, missing1, missing2, diff1, diff2 map[string]*memFSNode, ignoreModTime bool) {
+	compareNode(fs1.tree, fs2.tree, missing1, missing2, diff1, diff2, "/", ignoreModTime)
+	image1Format := image1Name.GetRepository() + ":" + image1Name.GetTag()
+	image2Format := image2Name.GetRepository() + ":" + image2Name.GetTag()
+	// Files missing in first image but appeared in second image
+	log.Infof("======== file missing in first image %s=====", image1Format)
+	for _, node := range missing1 {
+		logMemFSNodeInfo(node)
+	}
+
+	// Files missing in second image but appeared in first image.
+	log.Infof("========= file missing in second image %s=====", image2Format)
+	for _, node := range missing2 {
+		logMemFSNodeInfo(node)
+	}
+
+	// File differences in two images.
+	log.Infof("======== difference between two images %s and %s ======", image1Format, image2Format)
+	for path := range diff1 {
+		hdr1, hdr2 := diff1[path].hdr, diff2[path].hdr
+		log.Infof("%s %s %d %d %d", path, hdr1.FileInfo().Mode(), hdr1.Uid, hdr1.Gid, hdr1.Size)
+		log.Infof("%s %s %d %d %d", path, hdr2.FileInfo().Mode(), hdr2.Uid, hdr2.Gid, hdr2.Size)
+		log.Infof("xxxxxxxxxxxxxxxxxxxxxxxxxxx")
+	}
+}

lib/snapshot/mem_fs_test.go

@@ -20,8 +20,10 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"testing"
 
+	"github.com/uber/makisu/lib/docker/image"
 	"github.com/uber/makisu/lib/pathutils"
 
 	"github.com/andres-erbsen/clock"
@@ -1194,3 +1196,89 @@ func TestAddLayersEqual(t *testing.T) {
 	require.Equal(b3, b1)
 	require.Equal(b2, b1)
 }
+
+func TestCompareFS(t *testing.T) {
+	require := require.New(t)
+
+	tmpRoot, err := ioutil.TempDir("/tmp", "makisu-test")
+	require.NoError(err)
+	defer os.RemoveAll(tmpRoot)
+
+	clk := clock.NewMock()
+	fs1, err := NewMemFS(clk, tmpRoot, pathutils.DefaultBlacklist)
+	require.NoError(err)
+
+	l1 := newMemLayer()
+	dst11 := "/common"
+	require.NoError(addDirectoryToLayer(l1, tmpRoot, dst11, 0755))
+	dst12 := "/common/test1"
+	require.NoError(addDirectoryToLayer(l1, tmpRoot, dst12, 0755))
+	dst13 := "/common/world"
+	require.NoError(addRegularFileToLayer(l1, tmpRoot, dst13, "hello", 0711))
+	require.NoError(fs1.merge(l1))
+
+	fs2, err := NewMemFS(clk, tmpRoot, pathutils.DefaultBlacklist)
+	require.NoError(err)
+
+	l2 := newMemLayer()
+	dst21 := "/common"
+	require.NoError(addDirectoryToLayer(l2, tmpRoot, dst21, 0755))
+	dst22 := "/common/test2"
+	require.NoError(addDirectoryToLayer(l2, tmpRoot, dst22, 0755))
+	dst23 := "/common/world"
+	require.NoError(addRegularFileToLayer(l2, tmpRoot, dst23, "hello", 0755))
+	require.NoError(fs2.merge(l2))
+
+	image1Name := image.NewImageName("127.0.0.1:15055", "uber-usi/docker", "tag1")
+	image2Name := image.NewImageName("127.0.0.1:15055", "uber-usi/docker", "tag2")
+
+	missing1 := make(map[string]*memFSNode)
+	missing2 := make(map[string]*memFSNode)
+	diff1 := make(map[string]*memFSNode)
+	diff2 := make(map[string]*memFSNode)
+
+	compareFS(fs1, fs2, image1Name, image2Name, missing1, missing2, diff1, diff2, true)
+	// Check Missing paths.
+	expectedMissing1Nums := 1
+	require.Equal(expectedMissing1Nums, len(missing1))
+
+	expectedMissing1Paths := []string{"/common/test2"}
+	actualMissing1Paths := make([]string, 0)
+	for path1 := range missing1 {
+		actualMissing1Paths = append(actualMissing1Paths, path1)
+	}
+
+	sort.Strings(actualMissing1Paths)
+	sort.Strings(expectedMissing1Paths)
+	require.Equal(actualMissing1Paths, expectedMissing1Paths)
+
+	expectedMissing2Nums := 1
+	require.Equal(expectedMissing2Nums, len(missing2))
+	expectedMissing2Paths := []string{"/common/test1"}
+	actualMissing2Paths := make([]string, 0)
+	for path2 := range missing2 {
+		actualMissing2Paths = append(actualMissing2Paths, path2)
+	}
+
+	sort.Strings(actualMissing2Paths)
+	sort.Strings(expectedMissing2Paths)
+	require.Equal(actualMissing2Paths,
+		expectedMissing2Paths)
+
+	// Check diff paths.
+	expectedDiff := []string{"/common/world"}
+	actualDiff1 := make([]string, 0)
+	for path := range diff1 {
+		actualDiff1 = append(actualDiff1, path)
+	}
+
+	actualDiff2 := make([]string, 0)
+	for path := range diff2 {
+		actualDiff2 = append(actualDiff2, path)
+	}
+
+	sort.Strings(actualDiff1)
+	sort.Strings(actualDiff2)
+	require.Equal(expectedDiff, actualDiff1)
+	require.Equal(expectedDiff, actualDiff2)
+}

lib/snapshot/testutils_test.go

@@ -51,14 +51,14 @@ func requireEqualLayers(require *require.Assertions, l1 *memLayer, l2 *memLayer)
 			content2, ok := mf2.(*contentMemFile)
 			require.True(ok)
 			require.Equal(content1.dst, content2.dst, "%s dsts not equal:\nexpected=%s, actual=%s", p1, content1.dst, content2.dst)
-			similar, err := tario.IsSimilarHeader(content1.hdr, content2.hdr)
+			similar, err := tario.IsSimilarHeader(content1.hdr, content2.hdr, false)
 			require.NoError(err)
 			require.True(similar, "%s headers not similar:\nexpected=%+v\nactual=  %+v", p1, content1.hdr, content2.hdr)
 		} else if whiteout1, ok := mf1.(*whiteoutMemFile); ok {
 			whiteout2, ok := mf2.(*whiteoutMemFile)
 			require.True(ok)
 			require.Equal(whiteout1.del, whiteout2.del)
-			similar, err := tario.IsSimilarHeader(whiteout1.hdr, whiteout2.hdr)
+			similar, err := tario.IsSimilarHeader(whiteout1.hdr, whiteout2.hdr, false)
 			require.NoError(err)
 			require.True(similar)
 		} else {