gvfs-helper: add prefetch .keep file for last packfile

Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>

Author: jeffhostetler

gvfs-helper.c

@@ -1890,6 +1890,7 @@ static void my_run_index_pack(struct gh__request_params *params UNUSED,
 
 static void my_finalize_packfile(struct gh__request_params *params,
 				 struct gh__response_status *status,
+				 int b_keep,
 				 const struct strbuf *temp_path_pack,
 				 const struct strbuf *temp_path_idx,
 				 struct strbuf *final_path_pack,
@@ -1909,6 +1910,21 @@ static void my_finalize_packfile(struct gh__request_params *params,
 		return;
 	}
 
+	if (b_keep) {
+		struct strbuf keep = STRBUF_INIT;
+		int fd_keep;
+
+		strbuf_addbuf(&keep, final_path_pack);
+		strbuf_strip_suffix(&keep, ".pack");
+		strbuf_addstr(&keep, ".keep");
+
+		fd_keep = xopen(keep.buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
+		if (fd_keep >= 0)
+			close(fd_keep);
+
+		strbuf_release(&keep);
+	}
+
 	if (params->result_list) {
 		struct strbuf result_msg = STRBUF_INIT;
 
@@ -1961,7 +1977,7 @@ static void install_packfile(struct gh__request_params *params,
 	create_final_packfile_pathnames("vfs", packfile_checksum.buf, NULL,
 					&final_path_pack, &final_path_idx,
 					&final_filename);
-	my_finalize_packfile(params, status,
+	my_finalize_packfile(params, status, 0,
 			     &temp_path_pack, &temp_path_idx,
 			     &final_path_pack, &final_path_idx,
 			     &final_filename);
@@ -2057,6 +2073,12 @@ struct ph {
 
 /*
  * Extract the next packfile from the multipack.
+ * Install {.pack, .idx, .keep} set.
+ *
+ * Mark each successfully installed prefetch pack as .keep it as installed
+ * in case we have errors decoding/indexing later packs within the received
+ * multipart file.  (A later pass can delete the unnecessary .keep files
+ * from this and any previous invocations.)
  */
 static void extract_packfile_from_multipack(
 	struct gh__request_params *params,
@@ -2153,7 +2175,7 @@ static void extract_packfile_from_multipack(
 
 	} else {
 		/*
-		 * Server send the .idx immediately after the .pack in the
+		 * Server sent the .idx immediately after the .pack in the
 		 * data stream.  I'm tempted to verify it, but that defeats
 		 * the purpose of having it cached...
 		 */
@@ -2176,7 +2198,7 @@ static void extract_packfile_from_multipack(
 					&final_filename);
 	strbuf_release(&buf_timestamp);
 
-	my_finalize_packfile(params, status,
+	my_finalize_packfile(params, status, 1,
 			     &temp_path_pack, &temp_path_idx,
 			     &final_path_pack, &final_path_idx,
 			     &final_filename);
@@ -2191,6 +2213,56 @@ static void extract_packfile_from_multipack(
 	strbuf_release(&final_filename);
 }
 
+struct keep_files_data {
+	timestamp_t max_timestamp;
+	int pos_of_max;
+	struct string_list *keep_files;
+};
+
+static void cb_keep_files(const char *full_path, size_t full_path_len UNUSED,
+			  const char *file_path, void *void_data)
+{
+	struct keep_files_data *data = void_data;
+	const char *val;
+	timestamp_t t;
+
+	/*
+	 * We expect prefetch packfiles named like:
+	 *
+	 *     prefetch-<seconds>-<checksum>.keep
+	 */
+	if (!skip_prefix(file_path, "prefetch-", &val))
+		return;
+	if (!ends_with(val, ".keep"))
+		return;
+
+	t = strtol(val, NULL, 10);
+	if (t > data->max_timestamp) {
+		data->pos_of_max = data->keep_files->nr;
+		data->max_timestamp = t;
+	}
+
+	string_list_append(data->keep_files, full_path);
+}
+
+static void delete_stale_keep_files(
+	struct gh__request_params *params UNUSED,
+	struct gh__response_status *status UNUSED)
+{
+	struct string_list keep_files = STRING_LIST_INIT_DUP;
+	struct keep_files_data data = { 0, 0, &keep_files };
+	size_t k;
+
+	for_each_file_in_pack_dir(gh__global.buf_odb_path.buf,
+				  cb_keep_files, &data);
+	for (k = 0; k < keep_files.nr; k++) {
+		if ((ssize_t)k != data.pos_of_max)
+			unlink(keep_files.items[k].string);
+	}
+
+	string_list_clear(&keep_files, 0);
+}
+
 /*
  * Cut apart the received multipart response into individual packfiles
  * and install each one.
@@ -2209,6 +2281,7 @@ static void install_prefetch(struct gh__request_params *params,
 	unsigned short np;
 	unsigned short k;
 	int fd = -1;
+	int nr_installed = 0;
 
 	struct strbuf temp_path_mp = STRBUF_INIT;
 
@@ -2253,9 +2326,13 @@ static void install_prefetch(struct gh__request_params *params,
 		display_progress(params->progress, k + 1);
 		if (status->ec != GH__ERROR_CODE__OK)
 			break;
+		nr_installed++;
 	}
 	stop_progress(&params->progress);
 
+	if (nr_installed)
+		delete_stale_keep_files(params, status);
+
 cleanup:
 	if (fd != -1)
 		close(fd);

t/t5799-gvfs-helper.sh

@@ -390,6 +390,30 @@ verify_received_packfile_count () {
 	return 0
 }
 
+# Verify that we have exactly 1 prefetch .keep file.
+# Optionally, verify that it has the given timestamp.
+#
+verify_prefetch_keeps () {
+	count=$(( $(ls -1 "$SHARED_CACHE_T1"/pack/prefetch-*.keep | wc -l) ))
+	if test $count -ne 1
+	then
+		echo "verify_prefetch_keep_file_count: found $count, expected 1."
+		return 1
+	fi
+
+	if test $# -eq 1
+	then
+		count=$(( $(ls -1 "$SHARED_CACHE_T1"/pack/prefetch-$1-*.keep | wc -l) ))
+		if test $count -ne 1
+		then
+			echo "verify_prefetch_keep_file_count: did not find expected keep file."
+			return 1
+		fi
+	fi
+
+	return 0
+}
+
 per_test_cleanup () {
 	stop_gvfs_protocol_server
 
@@ -643,6 +667,7 @@ test_expect_success 'basic: PREFETCH w/o arg gets all' '
 	# packfile.
 	#
 	verify_received_packfile_count 3 &&
+	verify_prefetch_keeps 1200000000 &&
 
 	stop_gvfs_protocol_server &&
 	verify_connection_count 1
@@ -664,6 +689,7 @@ test_expect_success 'basic: PREFETCH w/ arg' '
 	# packfile.
 	#
 	verify_received_packfile_count 2 &&
+	verify_prefetch_keeps 1200000000 &&
 
 	stop_gvfs_protocol_server &&
 	verify_connection_count 1
@@ -686,6 +712,7 @@ test_expect_success 'basic: PREFETCH mayhem no_prefetch_idx' '
 	# packfile.
 	#
 	verify_received_packfile_count 2 &&
+	verify_prefetch_keeps 1200000000 &&
 
 	stop_gvfs_protocol_server &&
 	verify_connection_count 1
@@ -707,6 +734,7 @@ test_expect_success 'basic: PREFETCH up-to-date' '
 	# packfile.
 	#
 	verify_received_packfile_count 2 &&
+	verify_prefetch_keeps 1200000000 &&
 
 	# Ask again for any packfiles newer than what we have cached locally.
 	#
@@ -720,6 +748,7 @@ test_expect_success 'basic: PREFETCH up-to-date' '
 	# packfile.
 	#
 	verify_received_packfile_count 0 &&
+	verify_prefetch_keeps 1200000000 &&
 
 	stop_gvfs_protocol_server &&
 	verify_connection_count 2