sparse-checkout: clear tracked sparse dirs

When changing the scope of a sparse-checkout using cone mode, we might
have some tracked directories go out of scope. The current logic removes
the tracked files from within those directories, but leaves the ignored
files within those directories. This is a bit unexpected to users who
have given input to Git saying they don't need those directories
anymore.

Since these ignored files are typically build output or helper files
from IDEs, the users should not need the files now that the tracked
files are removed. If the tracked files reappear, then they will have
newer timestamps than the build artifacts, so the artifacts will need to
be regenerated anyway.

Leaving these ignored files in the sparse directories makes it
impossible to gain performance benefits in the sparse index. When we
track into these directories, we need to know if the files are ignored
or not, which might depend on the _tracked_ .gitignore file(s) within
the sparse directory. This depends on the indexed version of the file,
so the sparse directory must be expanded.

By deleting the sparse directories when changing scope (or running 'git
sparse-checkout reapply') we regain these performance benefits as if the
repository was in a clean state.

If users depend on ignored files within the sparse directories, then
they have created a bad shape in their repository. This shape makes it
impossible to get performance benefits using the sparse index, so they
can workaround it (currently) by disabling the sparse index.

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>

Author: derrickstolee

builtin/sparse-checkout.c

@@ -15,6 +15,7 @@
 #include "wt-status.h"
 #include "quote.h"
 #include "sparse-index.h"
+#include "run-command.h"
 
 static const char *empty_base = "";
 
@@ -100,6 +101,71 @@ static int sparse_checkout_list(int argc, const char **argv)
 	return 0;
 }
 
+static void clean_tracked_sparse_directories(struct repository *r)
+{
+	int i;
+	struct strvec args = STRVEC_INIT;
+
+	/*
+	 * If we are not using cone mode patterns, then we cannot
+	 * delete directories outside of the sparse cone.
+	 */
+	if (!r || !r->index || !r->index->sparse_checkout_patterns ||
+	    !r->index->sparse_checkout_patterns->use_cone_patterns)
+		return;
+	/*
+	 * NEEDSWORK: For now, only use this behavior when index.sparse
+	 * is enabled. We may want this behavior enabled whenever using
+	 * cone mode patterns.
+	 */
+	prepare_repo_settings(r);
+	if (!r->settings.sparse_index)
+		return;
+
+	strvec_pushl(&args, "clean", "-dfx", NULL);
+
+	/*
+	 * Since we now depend on the sparse index to enable this
+	 * behavior, use it to our advantage. This process is more
+	 * complicated without it.
+	 */
+	convert_to_sparse(r->index);
+
+	for (i = 0; i < r->index->cache_nr; i++) {
+		struct cache_entry *ce = r->index->cache[i];
+
+		/*
+		 * Is this a sparse directory? If so, then definitely
+		 * include it. All contained content is outside of the
+		 * patterns.
+		 */
+		if (S_ISSPARSEDIR(ce->ce_mode) &&
+		    repo_file_exists(r, ce->name)) {
+			strvec_push(&args, ce->name);
+			continue;
+		}
+	}
+
+	run_command_v_opt(args.v, RUN_GIT_CMD);
+
+	/*
+	 * The 'git clean -dfx -- <path> ...' command empties the
+	 * tracked directories outside of the sparse cone, but does not
+	 * delete the directories themselves. Remove them now.
+	 */
+	for (i = 2; i < args.nr; i++)
+		rmdir_or_warn(args.v[i]);
+
+	strvec_clear(&args);
+
+	/*
+	 * This is temporary: the sparse-checkout builtin is not
+	 * integrated with the sparse-index yet, so we need to keep
+	 * it full during the process.
+	 */
+	ensure_full_index(r->index);
+}
+
 static int update_working_directory(struct pattern_list *pl)
 {
 	enum update_sparsity_result result;
@@ -141,6 +207,8 @@ static int update_working_directory(struct pattern_list *pl)
 	else
 		rollback_lock_file(&lock_file);
 
+	clean_tracked_sparse_directories(r);
+
 	r->index->sparse_checkout_patterns = NULL;
 	return result;
 }
@@ -540,8 +608,11 @@ static int modify_pattern_list(int argc, const char **argv, enum modify_type m)
 {
 	int result;
 	int changed_config = 0;
+	struct pattern_list *old_pl = xcalloc(1, sizeof(*old_pl));
 	struct pattern_list *pl = xcalloc(1, sizeof(*pl));
 
+	get_sparse_checkout_patterns(old_pl);
+
 	switch (m) {
 	case ADD:
 		if (core_sparse_checkout_cone)
@@ -567,7 +638,9 @@ static int modify_pattern_list(int argc, const char **argv, enum modify_type m)
 		set_config(MODE_NO_PATTERNS);
 
 	clear_pattern_list(pl);
+	clear_pattern_list(old_pl);
 	free(pl);
+	free(old_pl);
 	return result;
 }
 

t/t1091-sparse-checkout-builtin.sh

@@ -642,4 +642,42 @@ test_expect_success MINGW 'cone mode replaces backslashes with slashes' '
 	check_files repo/deep a deeper1
 '
 
+test_expect_success 'cone mode clears ignored subdirectories' '
+	rm repo/.git/info/sparse-checkout &&
+
+	# NEEDSWORK: --sparse-index is required for now
+	git -C repo sparse-checkout init --cone --sparse-index &&
+	git -C repo sparse-checkout set deep/deeper1 &&
+
+	cat >repo/.gitignore <<-\EOF &&
+	obj/
+	*.o
+	EOF
+
+	git -C repo add .gitignore &&
+	git -C repo commit -m ".gitignore" &&
+
+	for dir in  folder1 obj folder1/obj deep/deeper2 deep/deeper2/obj
+	do
+		mkdir repo/$dir || return 1
+	done &&
+	for file in folder1/obj/a obj/a folder1/file.o folder1.o \
+		    deep/deeper2/obj/a deep/deeper2/file.o file.o
+	do
+		echo ignored >repo/$file || return 1
+	done &&
+
+	git -C repo status --porcelain=v2 >out &&
+	test_must_be_empty out &&
+
+	git -C repo sparse-checkout reapply &&
+	test_path_is_missing repo/folder1 &&
+	test_path_is_missing repo/deep/deeper2 &&
+	test_path_is_dir repo/obj &&
+	test_path_is_file repo/file.o &&
+
+	git -C repo status --porcelain=v2 >out &&
+	test_must_be_empty out
+'
+
 test_done

t/t1092-sparse-checkout-compatibility.sh

@@ -594,7 +594,7 @@ test_expect_success 'sparse-index is not expanded' '
 	ensure_not_expanded add .
 '
 
-test_expect_failure 'sparse-index is not expanded (with ignored files outside cone)' '
+test_expect_success 'sparse-index is not expanded (with ignored files outside cone)' '
 	init_repos &&
 
 	write_script adjust_repo <<-\EOF &&
@@ -606,6 +606,7 @@ test_expect_failure 'sparse-index is not expanded (with ignored files outside co
 	EOF
 
 	run_on_all ../adjust_repo &&
+	git -C sparse-index sparse-checkout reapply &&
 
 	ensure_not_expanded status &&
 	ensure_not_expanded commit --allow-empty -m empty &&
@@ -622,14 +623,7 @@ test_expect_failure 'sparse-index is not expanded (with ignored files outside co
 	git -C sparse-index reset --hard &&
 	ensure_not_expanded checkout rename-out-to-out -- deep/deeper1 &&
 	git -C sparse-index reset --hard &&
-	ensure_not_expanded restore -s rename-out-to-out -- deep/deeper1 &&
-
-	echo >>sparse-index/README.md &&
-	ensure_not_expanded add -A &&
-	echo >>sparse-index/extra.txt &&
-	ensure_not_expanded add extra.txt &&
-	echo >>sparse-index/untracked.txt &&
-	ensure_not_expanded add .
+	ensure_not_expanded restore -s rename-out-to-out -- deep/deeper1
 '
 
 test_expect_success 'reset mixed and checkout orphan' '