Used env vars to search for packages in PYTHONPATH

Author: cftorres

.gofmt.sh

@@ -1,4 +1,3 @@
-
 #!/bin/bash
 
 # Copyright 2017 Google Inc. All rights reserved.

differs/aptDiff.go

@@ -19,7 +19,8 @@ func (d AptDiffer) Diff(image1, image2 utils.Image) (utils.DiffResult, error) {
 	return diff, err
 }
 
-func (d AptDiffer) getPackages(path string) (map[string]utils.PackageInfo, error) {
+func (d AptDiffer) getPackages(image utils.Image) (map[string]utils.PackageInfo, error) {
+	path := image.FSPath
 	packages := make(map[string]utils.PackageInfo)
 	if _, err := os.Stat(path); err != nil {
 		// invalid image directory path

differs/aptDiff_test.go

@@ -105,7 +105,8 @@ func TestGetAptPackages(t *testing.T) {
 	}
 	for _, test := range testCases {
 		d := AptDiffer{}
-		packages, err := d.getPackages(test.path)
+		image := utils.Image{FSPath: test.path}
+		packages, err := d.getPackages(image)
 		if err != nil && !test.err {
 			t.Errorf("Got unexpected error: %s", err)
 		}

differs/nodeDiff.go

@@ -27,7 +27,8 @@ func buildNodePaths(path string) ([]string, error) {
 	return []string{globalPaths, localPath}, nil
 }
 
-func (d NodeDiffer) getPackages(path string) (map[string]map[string]utils.PackageInfo, error) {
+func (d NodeDiffer) getPackages(image utils.Image) (map[string]map[string]utils.PackageInfo, error) {
+	path := image.FSPath
 	packages := make(map[string]map[string]utils.PackageInfo)
 	if _, err := os.Stat(path); err != nil {
 		// path provided invalid

differs/nodeDiff_test.go

@@ -44,8 +44,9 @@ func TestGetNodePackages(t *testing.T) {
 	}
 
 	for _, test := range testCases {
+		image := utils.Image{FSPath: test.path}
 		d := NodeDiffer{}
-		packages, err := d.getPackages(test.path)
+		packages, err := d.getPackages(image)
 		if err != nil && !test.err {
 			t.Errorf("Got unexpected error: %s", err)
 		}

differs/package_differs.go

@@ -7,22 +7,19 @@ import (
 )
 
 type MultiVersionPackageDiffer interface {
-	getPackages(path string) (map[string]map[string]utils.PackageInfo, error)
+	getPackages(image utils.Image) (map[string]map[string]utils.PackageInfo, error)
 }
 
 type SingleVersionPackageDiffer interface {
-	getPackages(path string) (map[string]utils.PackageInfo, error)
+	getPackages(image utils.Image) (map[string]utils.PackageInfo, error)
 }
 
 func multiVersionDiff(image1, image2 utils.Image, differ MultiVersionPackageDiffer) (utils.DiffResult, error) {
-	img1FS := image1.FSPath
-	img2FS := image2.FSPath
-
-	pack1, err := differ.getPackages(img1FS)
+	pack1, err := differ.getPackages(image1)
 	if err != nil {
 		return &utils.MultiVersionPackageDiffResult{}, err
 	}
-	pack2, err := differ.getPackages(img2FS)
+	pack2, err := differ.getPackages(image2)
 	if err != nil {
 		return &utils.MultiVersionPackageDiffResult{}, err
 	}
@@ -33,14 +30,11 @@ func multiVersionDiff(image1, image2 utils.Image, differ MultiVersionPackageDiff
 }
 
 func singleVersionDiff(image1, image2 utils.Image, differ SingleVersionPackageDiffer) (utils.DiffResult, error) {
-	img1FS := image1.FSPath
-	img2FS := image2.FSPath
-
-	pack1, err := differ.getPackages(img1FS)
+	pack1, err := differ.getPackages(image1)
 	if err != nil {
 		return &utils.PackageDiffResult{}, err
 	}
-	pack2, err := differ.getPackages(img2FS)
+	pack2, err := differ.getPackages(image2)
 	if err != nil {
 		return &utils.PackageDiffResult{}, err
 	}

differs/pipDiff.go

@@ -3,8 +3,10 @@ package differs
 import (
 	"io/ioutil"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/GoogleCloudPlatform/container-diff/utils"
 	"github.com/golang/glog"
@@ -37,18 +39,43 @@ func getPythonVersion(pathToLayer string) ([]string, error) {
 	return matches, nil
 }
 
-func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.PackageInfo, error) {
-	packages := make(map[string]map[string]utils.PackageInfo)
+func getPythonPaths(vars []string) []string {
+	paths := []string{}
+	for _, envVar := range vars {
+		pythonPathPattern := regexp.MustCompile("^PYTHONPATH=(.*)")
+		match := pythonPathPattern.FindStringSubmatch(envVar)
+		if len(match) != 0 {
+			pythonPath := match[1]
+			paths = strings.Split(pythonPath, ":")
+			break
+		}
+	}
+	return paths
+}
 
+func (d PipDiffer) getPackages(image utils.Image) (map[string]map[string]utils.PackageInfo, error) {
+	path := image.FSPath
+	packages := make(map[string]map[string]utils.PackageInfo)
+	pythonPaths := []string{}
+	if !reflect.DeepEqual(utils.ConfigSchema{}, image.Config) {
+		paths := getPythonPaths(image.Config.Config.Env)
+		for _, p := range paths {
+			pythonPaths = append(pythonPaths, p)
+		}
+	}
 	pythonVersions, err := getPythonVersion(path)
 	if err != nil {
-		// layer doesn't have a Python version installed
+		// Image doesn't have Python installed
 		return packages, nil
 	}
 
-	for _, pyVersion := range pythonVersions {
-		packagesPath := filepath.Join(path, "usr/local/lib", pyVersion, "site-packages")
-		contents, err := ioutil.ReadDir(packagesPath)
+	for _, pythonVersion := range pythonVersions {
+		packagesPath := filepath.Join(path, "usr/local/lib", pythonVersion, "site-packages")
+		pythonPaths = append(pythonPaths, packagesPath)
+	}
+
+	for _, pythonPath := range pythonPaths {
+		contents, err := ioutil.ReadDir(pythonPath)
 		if err != nil {
 			// python version folder doesn't have a site-packages folder
 			continue
@@ -57,7 +84,6 @@ func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.Package
 		for i := 0; i < len(contents); i++ {
 			c := contents[i]
 			fileName := c.Name()
-
 			// check if package
 			packageDir := regexp.MustCompile("^([a-z|A-Z|0-9|_]+)-(([0-9]+?\\.){2,3})dist-info$")
 			packageMatch := packageDir.FindStringSubmatch(fileName)
@@ -69,7 +95,7 @@ func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.Package
 				// by taking the file entry alphabetically before it (for a package) or after it (for a script)
 				var size string
 				if i-1 >= 0 && contents[i-1].Name() == packageName {
-					packagePath := filepath.Join(packagesPath, packageName)
+					packagePath := filepath.Join(pythonPath, packageName)
 					intSize, err := utils.GetDirectorySize(packagePath)
 					if err != nil {
 						glog.Errorf("Could not obtain size for package %s", packagePath)
@@ -85,21 +111,21 @@ func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.Package
 					continue
 				}
 				currPackage := utils.PackageInfo{Version: version, Size: size}
-				addToMap(packages, packageName, pyVersion, currPackage)
+				addToMap(packages, packageName, pythonPath, currPackage)
 			}
 		}
 	}
 
 	return packages, nil
 }
 
-func addToMap(packages map[string]map[string]utils.PackageInfo, pack string, pyVersion string, packInfo utils.PackageInfo) {
+func addToMap(packages map[string]map[string]utils.PackageInfo, pack string, path string, packInfo utils.PackageInfo) {
 	if _, ok := packages[pack]; !ok {
 		// package not yet seen
 		infoMap := make(map[string]utils.PackageInfo)
-		infoMap[pyVersion] = packInfo
+		infoMap[path] = packInfo
 		packages[pack] = infoMap
 		return
 	}
-	packages[pack][pyVersion] = packInfo
+	packages[pack][path] = packInfo
 }

differs/pipDiff_test.go

@@ -53,46 +53,91 @@ func TestGetPythonVersion(t *testing.T) {
 			t.Error("Expected error but got none.")
 		}
 		if !reflect.DeepEqual(version, test.expectedVersions) {
-			t.Errorf("Expected: %s.  Got: %s", test.expectedVersions, version)
+			t.Errorf("\nExpected: %s\nGot: %s", test.expectedVersions, version)
 		}
 	}
 }
 
 func TestGetPythonPackages(t *testing.T) {
 	testCases := []struct {
-		path             string
+		descrip          string
+		image            utils.Image
 		expectedPackages map[string]map[string]utils.PackageInfo
 	}{
 		{
-			path:             "testDirs/pipTests/noPackagesTest",
+			descrip: "noPackagesTest",
+			image: utils.Image{
+				FSPath: "testDirs/pipTests/noPackagesTest",
+			},
 			expectedPackages: map[string]map[string]utils.PackageInfo{},
 		},
 		{
-			path: "testDirs/pipTests/packagesOneLayer",
+			descrip: "packagesMultiVersion, no PYTHONPATH",
+			image: utils.Image{
+				FSPath: "testDirs/pipTests/packagesMultiVersion",
+			},
+			expectedPackages: map[string]map[string]utils.PackageInfo{
+				"packageone": {
+					"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"},
+					"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python2.7/site-packages": {Version: "0.1.1", Size: "0"},
+				},
+				"packagetwo": {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
+				"script1":    {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "1.0", Size: "0"}},
+				"script2":    {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "2.0", Size: "0"}},
+				"script3":    {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python2.7/site-packages": {Version: "3.0", Size: "0"}},
+			},
+		},
+		{
+			descrip: "packagesSingleVersion, no PYTHONPATH",
+			image: utils.Image{
+				FSPath: "testDirs/pipTests/packagesSingleVersion",
+			},
 			expectedPackages: map[string]map[string]utils.PackageInfo{
-				"packageone": {"python3.6": {Version: "3.6.9", Size: "0"}},
-				"packagetwo": {"python3.6": {Version: "4.6.2", Size: "0"}},
-				"script1":    {"python3.6": {Version: "1.0", Size: "0"}},
-				"script2":    {"python3.6": {Version: "2.0", Size: "0"}},
+				"packageone": {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"}},
+				"packagetwo": {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
+				"script1":    {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "1.0", Size: "0"}},
+				"script2":    {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "2.0", Size: "0"}},
 			},
 		},
 		{
-			path: "testDirs/pipTests/packagesMultiVersion",
+			descrip: "pythonPathTests, PYTHONPATH",
+			image: utils.Image{
+				FSPath: "testDirs/pipTests/pythonPathTests",
+				Config: utils.ConfigSchema{
+					Config: utils.ConfigObject{
+						Env: []string{"PYTHONPATH=testDirs/pipTests/pythonPathTests/pythonPath1:testDirs/pipTests/pythonPathTests/pythonPath2/subdir", "ENVVAR2=something"},
+					},
+				},
+			},
+			expectedPackages: map[string]map[string]utils.PackageInfo{
+				"packageone":   {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"}},
+				"packagetwo":   {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
+				"packagefive":  {"testDirs/pipTests/pythonPathTests/pythonPath2/subdir": {Version: "3.6.9", Size: "0"}},
+				"packagesix":   {"testDirs/pipTests/pythonPathTests/pythonPath1": {Version: "3.6.9", Size: "0"}},
+				"packageseven": {"testDirs/pipTests/pythonPathTests/pythonPath1": {Version: "4.6.2", Size: "0"}},
+			},
+		},
+		{
+			descrip: "pythonPathTests, no PYTHONPATH",
+			image: utils.Image{
+				FSPath: "testDirs/pipTests/pythonPathTests",
+				Config: utils.ConfigSchema{
+					Config: utils.ConfigObject{
+						Env: []string{"ENVVAR=something"},
+					},
+				},
+			},
 			expectedPackages: map[string]map[string]utils.PackageInfo{
-				"packageone": {"python3.6": {Version: "3.6.9", Size: "0"},
-					"python2.7": {Version: "0.1.1", Size: "0"}},
-				"packagetwo": {"python3.6": {Version: "4.6.2", Size: "0"}},
-				"script1":    {"python3.6": {Version: "1.0", Size: "0"}},
-				"script2":    {"python3.6": {Version: "2.0", Size: "0"}},
-				"script3":    {"python2.7": {Version: "3.0", Size: "0"}},
+				"packageone": {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"}},
+				"packagetwo": {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
 			},
 		},
 	}
 	for _, test := range testCases {
 		d := PipDiffer{}
-		packages, _ := d.getPackages(test.path)
+		packages, _ := d.getPackages(test.image)
 		if !reflect.DeepEqual(packages, test.expectedPackages) {
-			t.Errorf("Expected: %s but got: %s", test.expectedPackages, packages)
+			t.Errorf("%s\nExpected: %s\nGot: %s", test.descrip, test.expectedPackages, packages)
 		}
 	}
 }

differs/testDirs/pipTests/noPackagesTest/noPackagesInstalled/usr/local/lib/python3.6/site-packages/file renamed to differs/testDirs/pipTests/noPackagesTest/noPackagesInstalled/layer/usr/local/lib/python3.6/site-packages/file

@@ -85,7 +85,7 @@
       "Image1": "gcr.io/gcp-runtimes/multi-base",
       "Packages1": {
         "pbr": {
-          "python3.6": {
+          "multi-base/usr/local/lib/python3.6/site-packages": {
             "Version": "3.1.1",
             "Size": "447110"
           }
@@ -94,25 +94,25 @@
       "Image2": "gcr.io/gcp-runtimes/multi-modified",
       "Packages2": {
         "retrying": {
-          "python3.6": {
+          "multi-modified/usr/local/lib/python3.6/site-packages": {
             "Version": "1.3.3",
             "Size": "9955"
           }
         }
       },
       "InfoDiff": [
         {
-          "Package": "mock",
+          "Package": "pip",
           "Info1": [
             {
-              "Version": "2.0.0",
-              "Size": "504226"
+              "Version": "9.0.1",
+              "Size": "5289421"
             }
           ],
           "Info2": [
             {
-              "Version": "0.8.0",
-              "Size": "73348"
+              "Version": "9.0.1",
+              "Size": "5289421"
             }
           ]
         },
@@ -131,6 +131,21 @@
             }
           ]
         },
+        {
+          "Package": "six",
+          "Info1": [
+            {
+              "Version": "1.10.0",
+              "Size": "30098"
+            }
+          ],
+          "Info2": [
+            {
+              "Version": "1.10.0",
+              "Size": "30098"
+            }
+          ]
+        },
         {
           "Package": "wheel",
           "Info1": [
@@ -145,6 +160,21 @@
               "Size": "137451"
             }
           ]
+        },
+        {
+          "Package": "mock",
+          "Info1": [
+            {
+              "Version": "2.0.0",
+              "Size": "504226"
+            }
+          ],
+          "Info2": [
+            {
+              "Version": "0.8.0",
+              "Size": "73348"
+            }
+          ]
         }
       ]
     }