Merge pull request microsoft#423 from vdye/sparse-index/update-index

Sparse index: integrate with `git update-index`

Author: vdye

builtin/update-index.c

@@ -410,6 +410,9 @@ static int add_cacheinfo(unsigned int mode, const struct object_id *oid,
 	if (!verify_path(path, mode))
 		return error("Invalid path '%s'", path);
 
+	if (S_ISSPARSEDIR(mode))
+		return error("%s: cannot add directory as cache entry", path);
+
 	len = strlen(path);
 	ce = make_empty_cache_entry(&the_index, len);
 
@@ -744,17 +747,23 @@ static int do_reupdate(int ac, const char **av,
 		 * commit.  Update everything in the index.
 		 */
 		has_head = 0;
+
  redo:
-	/* TODO: audit for interaction with sparse-index. */
-	ensure_full_index(&the_index);
 	for (pos = 0; pos < active_nr; pos++) {
 		const struct cache_entry *ce = active_cache[pos];
 		struct cache_entry *old = NULL;
 		int save_nr;
 		char *path;
 
-		if (ce_stage(ce) || !ce_path_match(&the_index, ce, &pathspec, NULL))
+		/*
+		 * We can safely skip re-updating sparse directories because if there
+		 * were any changes to re-update inside of the sparse directory, it
+		 * would not be sparse.
+		 */
+		if (S_ISSPARSEDIR(ce->ce_mode) || ce_stage(ce) ||
+		    !ce_path_match(&the_index, ce, &pathspec, NULL))
 			continue;
+
 		if (has_head)
 			old = read_one_ent(NULL, &head_oid,
 					   ce->name, ce_namelen(ce), 0);
@@ -1079,6 +1088,9 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
 
 	git_config(git_default_config, NULL);
 
+	prepare_repo_settings(r);
+	the_repository->settings.command_requires_full_index = 0;
+
 	/* we will diagnose later if it turns out that we need to update it */
 	newfd = hold_locked_index(&lock_file, 0);
 	if (newfd < 0)

read-cache.c

@@ -1352,9 +1352,6 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e
 	int skip_df_check = option & ADD_CACHE_SKIP_DFCHECK;
 	int new_only = option & ADD_CACHE_NEW_ONLY;
 
-	if (!(option & ADD_CACHE_KEEP_CACHE_TREE))
-		cache_tree_invalidate_path(istate, ce->name);
-
 	/*
 	 * If this entry's path sorts after the last entry in the index,
 	 * we can avoid searching for it.
@@ -1365,6 +1362,13 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e
 	else
 		pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce), EXPAND_SPARSE);
 
+	/*
+	 * Cache tree path should be invalidated only after index_name_stage_pos,
+	 * in case it expands a sparse index.
+	 */
+	if (!(option & ADD_CACHE_KEEP_CACHE_TREE))
+		cache_tree_invalidate_path(istate, ce->name);
+
 	/* existing match? Just replace it. */
 	if (pos >= 0) {
 		if (!new_only)

t/t1092-sparse-checkout-compatibility.sh

@@ -667,6 +667,156 @@ test_expect_success 'reset with wildcard pathspec' '
 	test_all_match git ls-files -s -- folder1
 '
 
+# NEEDSWORK: although update-index executes without error on files outside
+# the sparse checkout definition, it does not actually add the file to the
+# index. This is also true when "--no-ignore-skip-worktree-entries" is
+# specified.
+test_expect_success 'update-index add outside sparse definition' '
+	init_repos &&
+
+	write_script edit-contents <<-\EOF &&
+	echo text >>$1
+	EOF
+
+	run_on_sparse mkdir -p folder1 &&
+	run_on_sparse cp ../initial-repo/folder1/a folder1/a &&
+
+	# Edit the file only in sparse checkouts so that, when checking the status
+	# of the index, the unmodified full-checkout is compared to the "modified"
+	# sparse checkouts.
+	run_on_sparse ../edit-contents folder1/a &&
+
+	test_sparse_match git update-index --add folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+	test_sparse_match git update-index --add --no-ignore-skip-worktree-entries folder1/a &&
+	test_all_match git status --porcelain=v2
+'
+
+test_expect_success 'update-index remove outside sparse definition' '
+	init_repos &&
+
+	# When --remove is specified, files outside the sparse checkout definition
+	# are considered "removed".
+	rm -f full-checkout/folder1/a &&
+	test_all_match git update-index --remove folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	git reset --hard &&
+
+	# When --ignore-skip-worktree-entries is explicitly specified, a file
+	# outside the sparse definition is not added to the index as "removed"
+	# (thus matching the unmodified full-checkout).
+	test_sparse_match git update-index --remove --ignore-skip-worktree-entries folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	git reset --hard &&
+
+	# --force-remove supercedes --ignore-skip-worktree-entries and always
+	# removes the file from the index.
+	test_all_match git update-index --force-remove --ignore-skip-worktree-entries folder1/a &&
+	test_all_match git status --porcelain=v2
+'
+
+test_expect_success 'update-index folder add/remove' '
+	init_repos &&
+
+	test_all_match test_must_fail git update-index --add --remove deep &&
+	test_all_match test_must_fail git update-index --add --remove deep/ &&
+
+	# NEEDSWORK: attempting to update-index on an existing folder outside the
+	# sparse checkout definition does not throw an error (as it does for folders
+	# inside the definition, or in the full checkout). However, it is a no-op.
+	test_sparse_match git update-index --add --remove folder1 &&
+	test_sparse_match git update-index --add --remove folder1/ &&
+	test_sparse_match git update-index --force-remove folder1/ &&
+	test_all_match git status --porcelain=v2 &&
+
+	# New folders, even in sparse checkouts, throw an error on update-index
+	run_on_all mkdir folder3 &&
+	run_on_all cp a folder3/a &&
+	run_on_all test_must_fail git update-index --add --remove folder3
+'
+
+test_expect_success 'update-index with updated flags' '
+	init_repos &&
+
+	# NEEDSWORK: updating flags runs inconsistently on directories, performing no
+	# operation with warning text specifying the path being ignored if a trailing
+	# slash is in the path, but throwing an error if there is no trailing slash.
+	test_all_match test_must_fail git update-index --no-skip-worktree folder1 &&
+	test_all_match git update-index --no-skip-worktree folder1/ &&
+	test_all_match git status --porcelain=v2 &&
+
+	# Removing the skip-worktree bit from a file outside the sparse checkout
+	# will cause the file to appear as unstaged and deleted.
+	test_sparse_match git update-index --no-skip-worktree folder1/a &&
+	rm -f full-checkout/folder1/a &&
+	test_all_match git status --porcelain=v2
+'
+
+test_expect_success 'update-index --again file outside sparse definition' '
+	init_repos &&
+
+	write_script edit-contents <<-\EOF &&
+	echo text >>$1
+	EOF
+
+	# When a file is manually added and modified outside the checkout
+	# definition, it will not be changed with `--again` because its changes are
+	# not tracked in the index.
+	run_on_sparse mkdir -p folder1 &&
+	run_on_sparse ../edit-contents folder1/a &&
+	test_sparse_match git update-index --again &&
+	test_sparse_match git status --porcelain=v2 &&
+
+	# Update the sparse checkouts so that folder1/a is no longer skipped and
+	# exists on-disk
+	run_on_sparse cp ../initial-repo/folder1/a folder1/a &&
+	test_sparse_match git update-index --no-skip-worktree folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	# Stage change for commit
+	run_on_all ../edit-contents folder1/a &&
+	test_all_match git update-index folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	# Modify the file
+	run_on_all ../edit-contents folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	# Run update-index --again, which re-stages the local changes
+	test_all_match git update-index --again &&
+	test_all_match git ls-files -s folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	# Running update-index --again with staged changes after manually deleting
+	# the file on disk will cause it to fail if --remove is not also specified
+	run_on_all rm -f folder1/a &&
+	test_all_match test_must_fail git update-index --again folder1 &&
+	test_all_match git update-index --remove --again &&
+	test_all_match git status --porcelain=v2
+'
+
+test_expect_success 'update-index --cacheinfo' '
+	init_repos &&
+
+	deep_a_oid=$(git -C full-checkout rev-parse update-deep:deep/a) &&
+	folder2_oid=$(git -C full-checkout rev-parse update-folder2:folder2) &&
+	folder1_a_oid=$(git -C full-checkout rev-parse update-folder1:folder1/a) &&
+
+	test_all_match git update-index --cacheinfo 100644 $deep_a_oid deep/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	# Cannot add sparse directory, even in sparse index case
+	test_all_match test_must_fail git update-index --add --cacheinfo 040000 $folder2_oid folder2/ &&
+
+	# Sparse match only - because folder1/a is outside the sparse checkout
+	# definition (and thus not on-disk), it will appear as "deleted" in
+	# unstaged changes.
+	test_all_match git update-index --add --cacheinfo 100644 $folder1_a_oid folder1/a &&
+	test_sparse_match git status --porcelain=v2
+'
+
 test_expect_success 'merge, cherry-pick, and rebase' '
 	init_repos &&
 
@@ -1167,6 +1317,21 @@ test_expect_success 'sparse index is not expanded: sparse-checkout' '
 	ensure_not_expanded sparse-checkout set
 '
 
+test_expect_success 'sparse index is not expanded: update-index' '
+	init_repos &&
+
+	echo "test" >sparse-index/README.md &&
+	echo "test2" >sparse-index/a &&
+	rm -f sparse-index/deep/a &&
+
+	ensure_not_expanded update-index --add README.md &&
+	ensure_not_expanded update-index a &&
+	ensure_not_expanded update-index --remove deep/a &&
+
+	rm -f sparse-index/README.md sparse-index/a &&
+	ensure_not_expanded update-index --add --remove --again
+'
+
 # NEEDSWORK: a sparse-checkout behaves differently from a full checkout
 # in this scenario, but it shouldn't.
 test_expect_success 'reset mixed and checkout orphan' '

t/t2107-update-index-basic.sh

@@ -64,6 +64,14 @@ test_expect_success '--cacheinfo mode,sha1,path (new syntax)' '
 	test_cmp expect actual
 '
 
+test_expect_success '--cacheinfo does not accept directory mode' '
+	mkdir folder1 &&
+	echo content >folder1/content &&
+	git add folder1 &&
+	folder1_oid=$(git ls-files -s folder1 | git hash-object --stdin) &&
+	test_must_fail git update-index --add --cacheinfo 040000 $folder1_oid folder1/
+'
+
 test_expect_success '.lock files cleaned up' '
 	mkdir cleanup &&
 	(