Skip to content

Commit

Permalink
add diff cmd (uber-archive#327)
Browse files Browse the repository at this point in the history
* add diff cmd

* fix comments

* add TODO and remove py test for now
  • Loading branch information
jockeych authored Apr 30, 2020
1 parent 6c559ff commit 3d2c1bb
Show file tree
Hide file tree
Showing 7 changed files with 395 additions and 55 deletions.
104 changes: 104 additions & 0 deletions bin/makisu/cmd/diff.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package cmd

import (
"archive/tar"
"errors"
"fmt"
"os"

"github.com/andres-erbsen/clock"
"github.com/spf13/cobra"
"github.com/uber/makisu/lib/docker/image"
"github.com/uber/makisu/lib/log"
"github.com/uber/makisu/lib/registry"
"github.com/uber/makisu/lib/snapshot"
"github.com/uber/makisu/lib/storage"
"github.com/uber/makisu/lib/tario"
)

type diffCmd struct {
*cobra.Command
ignoreModTime bool
}

func getDiffCmd() *diffCmd {
diffCmd := &diffCmd{
Command: &cobra.Command{
Use: "diff <image name> <image name>",
DisableFlagsInUseLine: true,
Short: "Compare docker images from registry",
},
}

diffCmd.Args = func(cmd *cobra.Command, args []string) error {
if len(args) != 2 {
return errors.New("Requires two image names as arguments")
}
return nil
}

diffCmd.Run = func(cmd *cobra.Command, args []string) {
if err := diffCmd.Diff(args); err != nil {
log.Error(err)
os.Exit(1)
}
}

diffCmd.PersistentFlags().BoolVar(&diffCmd.ignoreModTime, "ignoreModTime", true, "Ignore mod time of image files when comparing images")
return diffCmd
}

func (cmd *diffCmd) Diff(imagesFullName []string) error {
log.Infof("ingore time? :%t", cmd.ignoreModTime)
var pullImages []image.Name
for _, imageFullName := range imagesFullName {
pullImage, err := image.ParseNameForPull(imageFullName)
if err != nil {
return fmt.Errorf("parse image %s: %s", pullImage, err)
}
pullImages = append(pullImages, pullImage)
}

if err := initRegistryConfig(""); err != nil {
return fmt.Errorf("failed to initialize registry configuration: %s", err)
}

store, err := storage.NewImageStore("/tmp/makisu-storage/")
if err != nil {
panic(err)
}

var memFSArr []*snapshot.MemFS
for i, pullImage := range pullImages {
client := registry.New(store, pullImage.GetRegistry(), pullImage.GetRepository())
manifest, err := client.Pull(pullImage.GetTag())
if err != nil {
panic(err)
}

memfs, err := snapshot.NewMemFS(clock.New(), "/tmp/makisu-storage/", nil)
if err != nil {
panic(err)
}

for _, descriptor := range manifest.Layers {
reader, err := store.Layers.GetStoreFileReader(descriptor.Digest.Hex())
if err != nil {
panic(fmt.Errorf("get reader from image %d layer: %s", i+1, err))
}
gzipReader, err := tario.NewGzipReader(reader)
if err != nil {
panic(fmt.Errorf("create gzip reader for layer: %s", err))
}
if err = memfs.UpdateFromTarReader(tar.NewReader(gzipReader), false); err != nil {
panic(fmt.Errorf("untar image %d layer reader: %s", i+1, err))
}
}
memFSArr = append(memFSArr, memfs)
}

log.Infof("* Diff two images")
// TODO: compare the image config.
snapshot.CompareFS(memFSArr[0], memFSArr[1], pullImages[0], pullImages[1], cmd.ignoreModTime)
return nil
}
1 change: 1 addition & 0 deletions bin/makisu/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func Execute() {
rootCmd.AddCommand(getVersionCmd())
rootCmd.AddCommand(getPullCmd().Command)
rootCmd.AddCommand(getPushCmd().Command)
rootCmd.AddCommand(getDiffCmd().Command)
if err := rootCmd.Execute(); err != nil {
log.Error(err)
os.Exit(1)
Expand Down
77 changes: 75 additions & 2 deletions lib/snapshot/mem_fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

"github.com/andres-erbsen/clock"
"github.com/uber/makisu/lib/docker/image"
"github.com/uber/makisu/lib/fileio"
"github.com/uber/makisu/lib/log"
"github.com/uber/makisu/lib/mountutils"
Expand Down Expand Up @@ -489,7 +490,7 @@ func (fs *MemFS) isUpdated(p string, hdr *tar.Header) (bool, *memFSNode, error)
}
}

similar, err := tario.IsSimilarHeader(curr.hdr, hdr)
similar, err := tario.IsSimilarHeader(curr.hdr, hdr, false)
if err != nil {
return false, nil, fmt.Errorf("compare header %s: %s", p, err)
}
Expand Down Expand Up @@ -599,7 +600,7 @@ func (fs *MemFS) untarOneItem(path string, header *tar.Header, r *tar.Reader) er
}

// If the file is already on disk, nothing needs to be done.
if similar, err := tario.IsSimilarHeader(localHeader, header); err != nil {
if similar, err := tario.IsSimilarHeader(localHeader, header, false); err != nil {
return fmt.Errorf("compare headers %s: %s", path, err)
} else if similar {
return nil
Expand Down Expand Up @@ -710,3 +711,75 @@ func (fs *MemFS) untarFile(path string, header *tar.Header, r *tar.Reader) error
}
return nil
}

// CompareFS is the public API for comparing merged layers of two images for differences.
func CompareFS(fs1, fs2 *MemFS, image1Name, image2Name image.Name, ignoreModTime bool) {
missing1 := make(map[string]*memFSNode)
missing2 := make(map[string]*memFSNode)
diff1 := make(map[string]*memFSNode)
diff2 := make(map[string]*memFSNode)

compareNode(fs1.tree, fs2.tree, missing1, missing2, diff1, diff2, "/", ignoreModTime)
image1Format := image1Name.GetRepository() + ":" + image1Name.GetTag()
image2Format := image2Name.GetRepository() + ":" + image2Name.GetTag()
// Files missing in first image but appeared in second image.
log.Infof("===== file missing in first image %s =====", image1Format)
for _, node := range missing1 {
logMemFSNodeInfo(node)
}

// Files missing in second image but appeared in first image.
log.Infof("===== file missing in second image %s =====", image2Format)
for _, node := range missing2 {
logMemFSNodeInfo(node)
}

// File differences in two images.
log.Infof("===== difference between two images %s and %s =====", image1Format, image2Format)
for path := range diff1 {
hdr1, hdr2 := diff1[path].hdr, diff2[path].hdr
log.Infof("%s %s %d %d %d", path, hdr1.FileInfo().Mode(), hdr1.Uid, hdr1.Gid, hdr1.Size)
log.Infof("%s %s %d %d %d", path, hdr2.FileInfo().Mode(), hdr2.Uid, hdr2.Gid, hdr2.Size)
log.Infof("xxxxxxxxxxxxxxxxxxxxxxxxxxx")
}
}

// compareNode compares two memFSNodes for differences.
func compareNode(node1, node2 *memFSNode, missing1, missing2, diff1, diff2 map[string]*memFSNode, path string, ignoreModTime bool) {
if isSimilar, _ := tario.IsSimilarHeader(node1.hdr, node2.hdr, ignoreModTime); !isSimilar {
diff1[path] = node1
diff2[path] = node2
}

allChildren := make(map[string]bool)
for child := range node1.children {
allChildren[child] = true
}

for child := range node2.children {
allChildren[child] = true
}

for child := range allChildren {
nextNode1, ok1 := node1.children[child]
nextNode2, ok2 := node2.children[child]
updatedPath := filepath.Join(path, child)
if ok1 && ok2 {
compareNode(nextNode1, nextNode2, missing1, missing2, diff1, diff2, updatedPath, ignoreModTime)
continue
} else if ok1 {
missing2[updatedPath] = node1.children[child]
} else if ok2 {
missing1[updatedPath] = node2.children[child]
}
}
}

//logMemFSNodeInfo logs the info of a memFSNode.
func logMemFSNodeInfo(node *memFSNode) {
hdr := node.hdr
log.Infof("%s %s %d %d %d", hdr.Name, hdr.FileInfo().Mode(), hdr.Uid, hdr.Gid, hdr.Size)
for _, nxtNode := range node.children {
logMemFSNodeInfo(nxtNode)
}
}
87 changes: 85 additions & 2 deletions lib/snapshot/mem_fs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ import (
"io/ioutil"
"os"
"path/filepath"
"sort"
"testing"

"github.com/uber/makisu/lib/pathutils"

"github.com/andres-erbsen/clock"
"github.com/stretchr/testify/require"
"github.com/uber/makisu/lib/pathutils"
)

func TestUntarFromPath(t *testing.T) {
Expand Down Expand Up @@ -1194,3 +1194,86 @@ func TestAddLayersEqual(t *testing.T) {
require.Equal(b3, b1)
require.Equal(b2, b1)
}

func TestCompareFS(t *testing.T) {
require := require.New(t)

tmpRoot, err := ioutil.TempDir("/tmp", "makisu-test")
require.NoError(err)
defer os.RemoveAll(tmpRoot)

clk := clock.NewMock()
fs1, err := NewMemFS(clk, tmpRoot, pathutils.DefaultBlacklist)
require.NoError(err)

l1 := newMemLayer()
dst11 := "/common"
require.NoError(addDirectoryToLayer(l1, tmpRoot, dst11, 0755))
dst12 := "/common/test1"
require.NoError(addDirectoryToLayer(l1, tmpRoot, dst12, 0755))
dst13 := "/common/world"
require.NoError(addRegularFileToLayer(l1, tmpRoot, dst13, "hello", 0711))
require.NoError(fs1.merge(l1))

fs2, err := NewMemFS(clk, tmpRoot, pathutils.DefaultBlacklist)
require.NoError(err)

l2 := newMemLayer()
dst21 := "/common"
require.NoError(addDirectoryToLayer(l2, tmpRoot, dst21, 0755))
dst22 := "/common/test2"
require.NoError(addDirectoryToLayer(l2, tmpRoot, dst22, 0755))
dst23 := "/common/world"
require.NoError(addRegularFileToLayer(l2, tmpRoot, dst23, "hello", 0755))
require.NoError(fs2.merge(l2))

missing1 := make(map[string]*memFSNode)
missing2 := make(map[string]*memFSNode)
diff1 := make(map[string]*memFSNode)
diff2 := make(map[string]*memFSNode)

compareNode(fs1.tree, fs2.tree, missing1, missing2, diff1, diff2, "/", true)
// Check Missing paths.
expectedMissing1Nums := 1
require.Equal(expectedMissing1Nums, len(missing1))

expectedMissing1Paths := []string{"/common/test2"}
actualMissing1Paths := make([]string, 0)
for path1 := range missing1 {
actualMissing1Paths = append(actualMissing1Paths, path1)
}

sort.Strings(actualMissing1Paths)
sort.Strings(expectedMissing1Paths)
require.Equal(actualMissing1Paths, expectedMissing1Paths)

expectedMissing2Nums := 1
require.Equal(expectedMissing2Nums, len(missing2))
expectedMissing2Paths := []string{"/common/test1"}
actualMissing2Paths := make([]string, 0)
for path2 := range missing2 {
actualMissing2Paths = append(actualMissing2Paths, path2)
}

sort.Strings(actualMissing2Paths)
sort.Strings(expectedMissing2Paths)
require.Equal(actualMissing2Paths,
expectedMissing2Paths)

// Check diff paths.
expectedDiff := []string{"/common/world"}
actualDiff1 := make([]string, 0)
for path := range diff1 {
actualDiff1 = append(actualDiff1, path)
}

actualDiff2 := make([]string, 0)
for path := range diff2 {
actualDiff2 = append(actualDiff2, path)
}

sort.Strings(actualDiff1)
sort.Strings(actualDiff2)
require.Equal(expectedDiff, actualDiff1)
require.Equal(expectedDiff, actualDiff2)
}
4 changes: 2 additions & 2 deletions lib/snapshot/testutils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,14 @@ func requireEqualLayers(require *require.Assertions, l1 *memLayer, l2 *memLayer)
content2, ok := mf2.(*contentMemFile)
require.True(ok)
require.Equal(content1.dst, content2.dst, "%s dsts not equal:\nexpected=%s, actual=%s", p1, content1.dst, content2.dst)
similar, err := tario.IsSimilarHeader(content1.hdr, content2.hdr)
similar, err := tario.IsSimilarHeader(content1.hdr, content2.hdr, false)
require.NoError(err)
require.True(similar, "%s headers not similar:\nexpected=%+v\nactual= %+v", p1, content1.hdr, content2.hdr)
} else if whiteout1, ok := mf1.(*whiteoutMemFile); ok {
whiteout2, ok := mf2.(*whiteoutMemFile)
require.True(ok)
require.Equal(whiteout1.del, whiteout2.del)
similar, err := tario.IsSimilarHeader(whiteout1.hdr, whiteout2.hdr)
similar, err := tario.IsSimilarHeader(whiteout1.hdr, whiteout2.hdr, false)
require.NoError(err)
require.True(similar)
} else {
Expand Down
Loading

0 comments on commit 3d2c1bb

Please sign in to comment.