Make scanner respect .gitignore files (google#191)

Closes google#165 

Also changes `GetCommitSHA` to no longer rely on the system's git
executable.

By default, OSV-Scanner will skip files/directories that are ignored by
a git project's `.gitignore` files. Added a flag `--no-ignore` to
disable this behaviour.

cmd/osv-scanner/fixtures/locks-gitignore/.gitignore

@@ -0,0 +1,3 @@
+ignored
+/yarn.lock
+composer*

cmd/osv-scanner/fixtures/locks-gitignore/Gemfile.lock

@@ -0,0 +1,16 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  ast
+
+RUBY VERSION
+   ruby 3.0.2p107
+
+BUNDLED WITH
+   2.2.28

cmd/osv-scanner/fixtures/locks-gitignore/ignored/Gemfile.lock

@@ -0,0 +1,16 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  ast
+
+RUBY VERSION
+   ruby 3.0.2p107
+
+BUNDLED WITH
+   2.2.28

cmd/osv-scanner/fixtures/locks-gitignore/ignored/yarn.lock

@@ -0,0 +1,7 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+balanced-match@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
+  integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==

cmd/osv-scanner/fixtures/locks-gitignore/subdir/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

cmd/osv-scanner/fixtures/locks-gitignore/subdir/Gemfile.lock

@@ -0,0 +1,16 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  ast
+
+RUBY VERSION
+   ruby 3.0.2p107
+
+BUNDLED WITH
+   2.2.28

cmd/osv-scanner/fixtures/locks-gitignore/subdir/yarn.lock

@@ -0,0 +1,7 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+balanced-match@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
+  integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==

cmd/osv-scanner/fixtures/locks-gitignore/yarn.lock

@@ -0,0 +1,7 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+balanced-match@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
+  integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==

cmd/osv-scanner/main.go

@@ -89,6 +89,11 @@ func run(args []string, stdout, stderr io.Writer) int {
 				Usage:   "check subdirectories",
 				Value:   false,
 			},
+			&cli.BoolFlag{
+				Name:  "no-ignore",
+				Usage: "also scan files that would be ignored by .gitignore",
+				Value: false,
+			},
 		},
 		ArgsUsage: "[directory1 directory2...]",
 		Action: func(context *cli.Context) error {
@@ -106,6 +111,7 @@ func run(args []string, stdout, stderr io.Writer) int {
 				DockerContainerNames: context.StringSlice("docker"),
 				Recursive:            context.Bool("recursive"),
 				SkipGit:              context.Bool("skip-git"),
+				NoIgnore:             context.Bool("no-ignore"),
 				ConfigOverridePath:   context.String("config"),
 				DirectoryPaths:       context.Args().Slice(),
 			}, r)

cmd/osv-scanner/main_test.go

@@ -194,6 +194,36 @@ func TestRun(t *testing.T) {
 			`,
 			wantStderr: "",
 		},
+		// .gitignored files
+		{
+			name:         "",
+			args:         []string{"", "--recursive", "./fixtures/locks-gitignore"},
+			wantExitCode: 0,
+			wantStdout: `
+				Scanning dir ./fixtures/locks-gitignore
+				Scanned %%/fixtures/locks-gitignore/Gemfile.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/subdir/yarn.lock file and found 1 packages
+			`,
+			wantStderr: "",
+		},
+		// ignoring .gitignore
+		{
+			name:         "",
+			args:         []string{"", "--recursive", "--no-ignore", "./fixtures/locks-gitignore"},
+			wantExitCode: 0,
+			wantStdout: `
+				Scanning dir ./fixtures/locks-gitignore
+				Scanned %%/fixtures/locks-gitignore/Gemfile.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/composer.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/ignored/Gemfile.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/ignored/yarn.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/subdir/Gemfile.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/subdir/composer.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/subdir/yarn.lock file and found 1 packages
+				Scanned %%/fixtures/locks-gitignore/yarn.lock file and found 1 packages
+			`,
+			wantStderr: "",
+		},
 		// output with json
 		{
 			name:         "",

go.mod

@@ -5,6 +5,8 @@ go 1.19
 require (
 	github.com/BurntSushi/toml v1.2.1
 	github.com/CycloneDX/cyclonedx-go v0.7.0
+	github.com/go-git/go-billy/v5 v5.4.0
+	github.com/go-git/go-git/v5 v5.5.2
 	github.com/google/go-cmp v0.5.9
 	github.com/jedib0t/go-pretty/v6 v6.4.4
 	github.com/package-url/packageurl-go v0.1.0
@@ -17,11 +19,27 @@ require (
 )
 
 require (
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/cloudflare/circl v1.1.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/pjbgf/sha1cd v0.2.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.1.0 // indirect
 	github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	golang.org/x/crypto v0.3.0 // indirect
+	golang.org/x/net v0.2.0 // indirect
 	golang.org/x/sys v0.4.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 )