Skip to content
This repository was archived by the owner on Mar 27, 2024. It is now read-only.

Commit 3a83f11

Browse files
abbytizabbytiz
authored
Multi version generalization (#12)
* Made node differ entries more comparable by narrowing the path used as keys in the map to package info structs, adjusting multiversion differ to be less node specific, still need to adjust unit and integration tests accordingly. * Unit tests passing * Integration test updates and pipDiffer bug fix (was returning if any pyVersion folder didn't have sitepackages rather than looking through all of them * travis typo * Adding .gofmt.sh file * Improving readability of diffMaps and multiVersionDiff
1 parent 1c449ea commit 3a83f11

11 files changed

+170
-183
lines changed

‎.gofmt.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
2+
#!/bin/bash
3+
4+
# Copyright 2017 Google Inc. All rights reserved.
5+
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
set -e
19+
20+
files=$(find . -name "*.go" | grep -v vendor/ | xargs gofmt -l -s)
21+
if [[ $files ]]; then
22+
echo "Gofmt errors in files: $files"
23+
exit 1
24+
fi

‎.travis.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@ go_import_path: github.com/GoogleCloudPlatform/container-diff
99
script:
1010
- ./.gofmt.sh
1111
- sudo rm -rf /etc/docker
12-
- travis_wait ./container-diff-tests.sh
12+
- travis_wait ./.container-diff-tests.sh

‎differs/nodeDiff.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,16 +61,16 @@ func (d NodeDiffer) getPackages(path string) (map[string]map[string]utils.Packag
6161
return packages, err
6262
}
6363
currInfo.Size = strconv.FormatInt(size, 10)
64-
64+
mapPath := strings.Replace(packagePath, path, "", 1)
6565
// Check if other package version already recorded
6666
if _, ok := packages[packageJSON.Name]; !ok {
6767
// package not yet seen
6868
infoMap := make(map[string]utils.PackageInfo)
69-
infoMap[currPackage] = currInfo
69+
infoMap[mapPath] = currInfo
7070
packages[packageJSON.Name] = infoMap
7171
continue
7272
}
73-
packages[packageJSON.Name][currPackage] = currInfo
73+
packages[packageJSON.Name][mapPath] = currInfo
7474

7575
}
7676
}

‎differs/nodeDiff_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,17 +29,17 @@ func TestGetNodePackages(t *testing.T) {
2929
descrip: "all packages in one layer",
3030
path: "testDirs/packageOne",
3131
expected: map[string]map[string]utils.PackageInfo{
32-
"pac1": {"testDirs/packageOne/node_modules/pac1/package.json": {Version: "1.0", Size: "41"}},
33-
"pac2": {"testDirs/packageOne/usr/local/lib/node_modules/pac2/package.json": {Version: "2.0", Size: "41"}},
34-
"pac3": {"testDirs/packageOne/node_modules/pac3/package.json": {Version: "3.0", Size: "41"}}},
32+
"pac1": {"/node_modules/pac1/": {Version: "1.0", Size: "41"}},
33+
"pac2": {"/usr/local/lib/node_modules/pac2/": {Version: "2.0", Size: "41"}},
34+
"pac3": {"/node_modules/pac3/": {Version: "3.0", Size: "41"}}},
3535
},
3636
{
3737
descrip: "Multi version packages",
3838
path: "testDirs/packageMulti",
3939
expected: map[string]map[string]utils.PackageInfo{
40-
"pac1": {"testDirs/packageMulti/node_modules/pac1/package.json": {Version: "1.0", Size: "41"}},
41-
"pac2": {"testDirs/packageMulti/node_modules/pac2/package.json": {Version: "2.0", Size: "41"},
42-
"testDirs/packageMulti/usr/local/lib/node_modules/pac2/package.json": {Version: "3.0", Size: "41"}}},
40+
"pac1": {"/node_modules/pac1/": {Version: "1.0", Size: "41"}},
41+
"pac2": {"/node_modules/pac2/": {Version: "2.0", Size: "41"},
42+
"/usr/local/lib/node_modules/pac2/": {Version: "3.0", Size: "41"}}},
4343
},
4444
}
4545

‎differs/pipDiff.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.Package
5050
packagesPath := filepath.Join(path, "usr/local/lib", pyVersion, "site-packages")
5151
contents, err := ioutil.ReadDir(packagesPath)
5252
if err != nil {
53-
// layer's Python folder doesn't have a site-packages folder
54-
return packages, nil
53+
// python version folder doesn't have a site-packages folder
54+
continue
5555
}
5656

5757
for i := 0; i < len(contents); i++ {

‎tests/multi_diff_expected.json

Lines changed: 54 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
"Image2": "gcr.io/gcp-runtimes/multi-modified",
5555
"Packages2": {
5656
"pax": {
57-
"multi-modified/node_modules/pax/package.json": {
57+
"/node_modules/pax/": {
5858
"Version": "0.2.1",
5959
"Size": "11998"
6060
}
@@ -84,46 +84,69 @@
8484
"Diff": {
8585
"Image1": "gcr.io/gcp-runtimes/multi-base",
8686
"Packages1": {
87-
"mock": {
88-
"python3.6": {
89-
"Version": "2.0.0",
90-
"Size": "504226"
91-
}
92-
},
9387
"pbr": {
9488
"python3.6": {
9589
"Version": "3.1.1",
9690
"Size": "447110"
9791
}
98-
},
99-
"pip": {
100-
"python3.6": {
101-
"Version": "9.0.1",
102-
"Size": "5289421"
103-
}
104-
},
105-
"setuptools": {
92+
}
93+
},
94+
"Image2": "gcr.io/gcp-runtimes/multi-modified",
95+
"Packages2": {
96+
"retrying": {
10697
"python3.6": {
107-
"Version": "36.2.2",
108-
"Size": "839895"
98+
"Version": "1.3.3",
99+
"Size": "9955"
109100
}
101+
}
102+
},
103+
"InfoDiff": [
104+
{
105+
"Package": "mock",
106+
"Info1": [
107+
{
108+
"Version": "2.0.0",
109+
"Size": "504226"
110+
}
111+
],
112+
"Info2": [
113+
{
114+
"Version": "0.8.0",
115+
"Size": "73348"
116+
}
117+
]
110118
},
111-
"six": {
112-
"python3.6": {
113-
"Version": "1.10.0",
114-
"Size": "30098"
115-
}
119+
{
120+
"Package": "setuptools",
121+
"Info1": [
122+
{
123+
"Version": "36.2.2",
124+
"Size": "839895"
125+
}
126+
],
127+
"Info2": [
128+
{
129+
"Version": "36.2.2",
130+
"Size": "1157078"
131+
}
132+
]
116133
},
117-
"wheel": {
118-
"python3.6": {
119-
"Version": "0.29.0",
120-
"Size": "103509"
121-
}
134+
{
135+
"Package": "wheel",
136+
"Info1": [
137+
{
138+
"Version": "0.29.0",
139+
"Size": "103509"
140+
}
141+
],
142+
"Info2": [
143+
{
144+
"Version": "0.29.0",
145+
"Size": "137451"
146+
}
147+
]
122148
}
123-
},
124-
"Image2": "gcr.io/gcp-runtimes/multi-modified",
125-
"Packages2": {},
126-
"InfoDiff": []
149+
]
127150
}
128151
}
129152
]

‎tests/node_diff_expected.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"Image2": "gcr.io/gcp-runtimes/node-modified",
88
"Packages2": {
99
"pax": {
10-
"node-modified/node_modules/pax/package.json": {
10+
"/node_modules/pax/": {
1111
"Version": "0.2.1",
1212
"Size": "11365"
1313
}

‎tests/node_diff_order_expected.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"Image2": "gcr.io/gcp-runtimes/node-modified",
88
"Packages2": {
99
"pax": {
10-
"node-modified/node_modules/pax/package.json": {
10+
"/node_modules/pax/": {
1111
"Version": "0.2.1",
1212
"Size": "11365"
1313
}

‎tests/pip_diff_expected.json

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,38 @@
2525
}
2626
}
2727
},
28-
"InfoDiff": []
28+
"InfoDiff": [
29+
{
30+
"Package": "pip",
31+
"Info1": [
32+
{
33+
"Version": "9.0.1",
34+
"Size": "3741310"
35+
}
36+
],
37+
"Info2": [
38+
{
39+
"Version": "9.0.1",
40+
"Size": "5289421"
41+
}
42+
]
43+
},
44+
{
45+
"Package": "wheel",
46+
"Info1": [
47+
{
48+
"Version": "0.29.0",
49+
"Size": "103348"
50+
}
51+
],
52+
"Info2": [
53+
{
54+
"Version": "0.29.0",
55+
"Size": "103509"
56+
}
57+
]
58+
}
59+
]
2960
}
3061
}
3162
]

‎utils/package_diff_utils.go

Lines changed: 29 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"io/ioutil"
66
"path/filepath"
77
"reflect"
8-
"strings"
98

109
"github.com/golang/glog"
1110
)
@@ -48,63 +47,30 @@ type PackageInfo struct {
4847
Size string
4948
}
5049

51-
func contains(info1 []PackageInfo, keys1 []string, key string, value PackageInfo) (int, bool) {
52-
if len(info1) != len(keys1) {
53-
return 0, false
54-
}
55-
for i, currVal := range info1 {
56-
if !reflect.DeepEqual(currVal, value) {
57-
continue
58-
}
59-
// Check if both global or local installations by trimming img-id and layer hash
60-
tempPath1 := strings.SplitN(key, "/", 3)
61-
tempPath2 := strings.SplitN(keys1[i], "/", 3)
62-
if len(tempPath1) != 3 || len(tempPath2) != 3 {
63-
continue
64-
}
65-
if tempPath1[2] == tempPath2[2] {
66-
return i, true
67-
}
68-
}
69-
return 0, false
70-
}
71-
72-
func multiVersionDiff(infoDiff []MultiVersionInfo, key string, map1, map2 map[string]PackageInfo) []MultiVersionInfo {
73-
diff1Possible := []PackageInfo{}
74-
diff1PossibleKeys := []string{}
75-
76-
for key1, value1 := range map1 {
77-
_, ok := map2[key1]
78-
if !ok {
79-
diff1Possible = append(diff1Possible, value1)
80-
diff1PossibleKeys = append(diff1PossibleKeys, key1)
81-
} else {
82-
// if key in both maps, means layer hash is the same therefore packages are the same
83-
delete(map2, key1)
84-
}
85-
}
86-
50+
func multiVersionDiff(infoDiff []MultiVersionInfo, packageName string, map1, map2 map[string]PackageInfo) []MultiVersionInfo {
8751
diff1 := []PackageInfo{}
8852
diff2 := []PackageInfo{}
89-
for key2, value2 := range map2 {
90-
index, ok := contains(diff1Possible, diff1PossibleKeys, key2, value2)
53+
for path, packInfo1 := range map1 {
54+
packInfo2, ok := map2[path]
9155
if !ok {
92-
diff2 = append(diff2, value2)
56+
diff1 = append(diff1, packInfo1)
57+
continue
9358
} else {
94-
if index == 0 {
95-
diff1Possible = diff1Possible[1:]
96-
diff1PossibleKeys = diff1PossibleKeys[1:]
59+
if reflect.DeepEqual(packInfo1, packInfo2) {
60+
delete(map2, path)
61+
} else {
62+
diff1 = append(diff1, packInfo1)
63+
diff2 = append(diff2, packInfo2)
64+
delete(map2, path)
9765
}
98-
diff1Possible = append(diff1Possible[:index], diff1Possible[index:]...)
99-
diff1PossibleKeys = append(diff1PossibleKeys[:index], diff1PossibleKeys[index:]...)
10066
}
10167
}
102-
103-
for _, val := range diff1Possible {
104-
diff1 = append(diff1, val)
68+
for _, packInfo2 := range map2 {
69+
diff2 = append(diff2, packInfo2)
10570
}
71+
10672
if len(diff1) > 0 || len(diff2) > 0 {
107-
infoDiff = append(infoDiff, MultiVersionInfo{key, diff1, diff2})
73+
infoDiff = append(infoDiff, MultiVersionInfo{packageName, diff1, diff2})
10874
}
10975
return infoDiff
11076
}
@@ -166,27 +132,27 @@ func diffMaps(map1, map2 interface{}) interface{} {
166132
infoDiff := []Info{}
167133
multiInfoDiff := []MultiVersionInfo{}
168134

169-
for _, key1 := range map1Value.MapKeys() {
170-
value1 := map1Value.MapIndex(key1)
171-
value2 := map2Value.MapIndex(key1)
172-
if !value2.IsValid() {
173-
diff1.SetMapIndex(key1, value1)
174-
} else if !reflect.DeepEqual(value2.Interface(), value1.Interface()) {
135+
for _, pack := range map1Value.MapKeys() {
136+
packageEntry1 := map1Value.MapIndex(pack)
137+
packageEntry2 := map2Value.MapIndex(pack)
138+
if !packageEntry2.IsValid() {
139+
diff1.SetMapIndex(pack, packageEntry1)
140+
} else if !reflect.DeepEqual(packageEntry2.Interface(), packageEntry1.Interface()) {
175141
if multiV {
176-
multiInfoDiff = multiVersionDiff(multiInfoDiff, key1.String(),
177-
value1.Interface().(map[string]PackageInfo), value2.Interface().(map[string]PackageInfo))
142+
multiInfoDiff = multiVersionDiff(multiInfoDiff, pack.String(),
143+
packageEntry1.Interface().(map[string]PackageInfo), packageEntry2.Interface().(map[string]PackageInfo))
178144
} else {
179-
infoDiff = append(infoDiff, Info{key1.String(), value1.Interface().(PackageInfo),
180-
value2.Interface().(PackageInfo)})
145+
infoDiff = append(infoDiff, Info{pack.String(), packageEntry1.Interface().(PackageInfo),
146+
packageEntry2.Interface().(PackageInfo)})
181147
}
182-
map2Value.SetMapIndex(key1, reflect.Value{})
148+
map2Value.SetMapIndex(pack, reflect.Value{})
183149
} else {
184-
map2Value.SetMapIndex(key1, reflect.Value{})
150+
map2Value.SetMapIndex(pack, reflect.Value{})
185151
}
186152
}
187153
for _, key2 := range map2Value.MapKeys() {
188-
value2 := map2Value.MapIndex(key2)
189-
diff2.SetMapIndex(key2, value2)
154+
packageEntry2 := map2Value.MapIndex(key2)
155+
diff2.SetMapIndex(key2, packageEntry2)
190156
}
191157
if multiV {
192158
return MultiVersionPackageDiff{Packages1: diff1.Interface().(map[string]map[string]PackageInfo),

0 commit comments

Comments
 (0)