enhance: add functionality to only checkout a subdirectory of the repo (hashicorp#1)

This is useful for module references like: `github.com/terraform-aws-modules/terraform-aws-rds//modules/db_instance`,
and means the whole repository doesn't need to be pulled if just a subdirectory is needed.

This doesn't handle any references within that submodule. For example, if it references modules located in other directories in the repo or if it symlinks into any other directories in the repo.

Author: aliscott

get_git.go

@@ -64,8 +64,9 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 	}
 
 	// Extract some query parameters we use
-	var ref, sshKey string
+	var ref, sshKey, subdir string
 	depth := 0 // 0 means "don't use shallow clone"
+
 	q := u.Query()
 	if len(q) > 0 {
 		ref = q.Get("ref")
@@ -74,6 +75,12 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 		sshKey = q.Get("sshkey")
 		q.Del("sshkey")
 
+		subdir = q.Get("subdir")
+		q.Del("subdir")
+		if subdir != "" {
+			depth = 1
+		}
+
 		if n, err := strconv.Atoi(q.Get("depth")); err == nil {
 			depth = n
 		}
@@ -127,7 +134,7 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 	if err == nil {
 		err = g.update(ctx, dst, sshKeyFile, u, ref, depth)
 	} else {
-		err = g.clone(ctx, dst, sshKeyFile, u, ref, depth)
+		err = g.clone(ctx, dst, sshKeyFile, u, ref, depth, subdir)
 	}
 	if err != nil {
 		return err
@@ -189,17 +196,27 @@ func (g *GitGetter) checkout(ctx context.Context, dst string, ref string) error
 // positives on short branch names that happen to also be "hex words".
 var gitCommitIDRegex = regexp.MustCompile("^[0-9a-fA-F]{7,40}$")
 
-func (g *GitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.URL, ref string, depth int) error {
+func (g *GitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.URL, ref string, depth int, subdir string) error {
 	args := []string{"clone"}
 
+	isCommitID := gitCommitIDRegex.MatchString(ref)
+
 	originalRef := ref // we handle an unspecified ref differently than explicitly selecting the default branch below
 	if ref == "" {
 		ref = findRemoteDefaultBranch(ctx, u)
 	}
 	if depth > 0 {
 		args = append(args, "--depth", strconv.Itoa(depth))
-		args = append(args, "--branch", ref)
+		if subdir == "" || !isCommitID {
+			args = append(args, "--branch", ref)
+		}
 	}
+	if subdir != "" {
+		args = append(args, "--filter=blob:none")
+		args = append(args, "--sparse")
+		args = append(args, "--no-checkout")
+	}
+
 	args = append(args, "--", u.String(), dst)
 
 	cmd := exec.CommandContext(ctx, "git", args...)
@@ -212,13 +229,33 @@ func (g *GitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.UR
 			// We can't accurately recognize the resulting error here without
 			// hard-coding assumptions about git's human-readable output, but
 			// we can at least try a heuristic.
-			if gitCommitIDRegex.MatchString(originalRef) {
+			if isCommitID {
 				return fmt.Errorf("%w (note that setting 'depth' requires 'ref' to be a branch or tag name)", err)
 			}
 		}
 		return err
 	}
 
+	if subdir != "" {
+		cmd = exec.CommandContext(ctx, "git", "sparse-checkout", "set", subdir)
+		cmd.Dir = dst
+		err = getRunCommand(cmd)
+		if err != nil {
+			return err
+		}
+
+		if isCommitID {
+			cmd = exec.CommandContext(ctx, "git", "fetch", "origin", ref, "--depth", "1")
+			cmd.Dir = dst
+			err = getRunCommand(cmd)
+			if err != nil {
+				return err
+			}
+		}
+
+		return g.checkout(ctx, dst, ref)
+	}
+
 	if depth < 1 && originalRef != "" {
 		// If we didn't add --depth and --branch above then we will now be
 		// on the remote repository's default branch, rather than the selected

get_git_test.go

@@ -892,7 +892,7 @@ func TestGitGetter_BadGitConfig(t *testing.T) {
 		err = g.update(ctx, dst, testGitToken, url, "main", 1)
 	} else {
 		// Clone a repository with a git config file
-		err = g.clone(ctx, dst, testGitToken, url, "main", 1)
+		err = g.clone(ctx, dst, testGitToken, url, "main", 1, "")
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
@@ -950,7 +950,7 @@ func TestGitGetter_BadGitDirName(t *testing.T) {
 		}
 	} else {
 		// Clone a repository with a git directory
-		err = g.clone(ctx, dst, testGitToken, url, "main", 1)
+		err = g.clone(ctx, dst, testGitToken, url, "main", 1, "")
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
@@ -984,6 +984,77 @@ func TestGitGetter_BadGitDirName(t *testing.T) {
 	}
 }
 
+func TestGitGetter_sparseCheckout(t *testing.T) {
+	if !testHasGit {
+		t.Skip("git not found, skipping")
+	}
+
+	g := new(GitGetter)
+	dst := tempDir(t)
+
+	repo := testGitRepo(t, "sparse-checkout")
+	repo.commitFile("subdir1/file1.txt", "hello")
+	repo.commitFile("subdir2/file2.txt", "world")
+
+	q := repo.url.Query()
+	q.Add("subdir", "subdir1")
+	repo.url.RawQuery = q.Encode()
+
+	if err := g.Get(dst, repo.url); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the file in subdir1 exists
+	mainPath := filepath.Join(dst, "subdir1/file1.txt")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the file in subdir2 does not exist
+	mainPath = filepath.Join(dst, "subdir2/file2.txt")
+	if _, err := os.Stat(mainPath); err == nil {
+		t.Fatalf("expected subdir2 file to not exist")
+	}
+}
+
+func TestGitGetter_sparseCheckoutWithCommitID(t *testing.T) {
+	if !testHasGit {
+		t.Skip("git not found, skipping")
+	}
+
+	g := new(GitGetter)
+	dst := tempDir(t)
+
+	repo := testGitRepo(t, "sparse-checkout-commit-id")
+	repo.commitFile("subdir1/file1.txt", "hello")
+	repo.commitFile("subdir2/file2.txt", "world")
+	commitID, err := repo.latestCommit()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	q := repo.url.Query()
+	q.Add("ref", commitID)
+	q.Add("subdir", "subdir1")
+	repo.url.RawQuery = q.Encode()
+
+	if err := g.Get(dst, repo.url); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the file in subdir1 exists
+	mainPath := filepath.Join(dst, "subdir1/file1.txt")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the file in subdir2 does not exist
+	mainPath = filepath.Join(dst, "subdir2/file2.txt")
+	if _, err := os.Stat(mainPath); err == nil {
+		t.Fatalf("expected subdir2 file to not exist")
+	}
+}
+
 // gitRepo is a helper struct which controls a single temp git repo.
 type gitRepo struct {
 	t   *testing.T
@@ -1035,6 +1106,13 @@ func (r *gitRepo) git(args ...string) {
 // commitFile writes and commits a text file to the repo.
 func (r *gitRepo) commitFile(file, content string) {
 	path := filepath.Join(r.dir, file)
+
+	// Ensure the directory structure exists
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		r.t.Fatal(err)
+	}
+
 	if err := ioutil.WriteFile(path, []byte(content), 0600); err != nil {
 		r.t.Fatal(err)
 	}