Skip to content

Commit ecaa06b

Browse files
mjcheethammjcheetham
committed
maintenance: add new vfs-cache-move maintenance task
Introduce a new maintenance task, `vfs-cache-move`, that operates on Scalar or VFS for Git repositories with a per-volume, shared object cache (specified by `gvfs.sharedCache`) to migrate packfiles from the repository object directory to the shared cache. Older versions of `microsoft/git` incorrectly placed packfiles in the repository object directory instead of the shared cache; this task will help clean up existing clones impacted by that issue. Signed-off-by: Matthew John Cheetham <mjcheetham@outlook.com>
1 parent df789d1 commit ecaa06b

File tree

3 files changed

+265
-0
lines changed

3 files changed

+265
-0
lines changed

‎Documentation/git-maintenance.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ task:
6969
* `prefetch`: hourly.
7070
* `loose-objects`: daily.
7171
* `incremental-repack`: daily.
72+
* `vfs-cache-move`: weekly.
7273
--
7374
+
7475
`git maintenance register` will also disable foreground maintenance by
@@ -158,6 +159,13 @@ pack-refs::
158159
need to iterate across many references. See linkgit:git-pack-refs[1]
159160
for more information.
160161

162+
vfs-cache-move::
163+
The `vfs-cache-move` task only operates on Scalar or VFS for Git
164+
repositories (cloned with either `scalar clone` or `gvfs clone`) that
165+
have the `gvfs.sharedCache` configuration setting present. This task
166+
migrates pack files from the repository's object directory in to the
167+
shared volume cache.
168+
161169
OPTIONS
162170
-------
163171
--auto::

‎builtin/gc.c‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
* Copyright (c) 2006 Shawn O. Pearce
1111
*/
1212
#define USE_THE_REPOSITORY_VARIABLE
13+
#include "git-compat-util.h"
1314
#include "builtin.h"
1415
#include "abspath.h"
1516
#include "date.h"
@@ -41,6 +42,8 @@
4142
#include "hook.h"
4243
#include "setup.h"
4344
#include "trace2.h"
45+
#include "copy.h"
46+
#include "dir.h"
4447

4548
#define FAILED_RUN "failed to run %s"
4649

@@ -1347,6 +1350,128 @@ static int maintenance_task_incremental_repack(struct maintenance_run_opts *opts
13471350
return 0;
13481351
}
13491352

1353+
static void link_or_copy_or_die(const char *src, const char *dst)
1354+
{
1355+
if (!link(src, dst))
1356+
return;
1357+
1358+
warning_errno(_("failed to link '%s' to '%s'"), src, dst);
1359+
1360+
if (!copy_file(dst, src, 0644))
1361+
return;
1362+
1363+
die_errno(_("failed to copy '%s' to '%s'"), src, dst);
1364+
}
1365+
1366+
static void migrate_pack(const char *srcdir, const char *dstdir,
1367+
const char *pack_filename)
1368+
{
1369+
struct stat st;
1370+
char *basename;
1371+
struct strbuf src = STRBUF_INIT, dst = STRBUF_INIT;
1372+
struct {
1373+
const char *ext;
1374+
unsigned move:1;
1375+
} files[] = {
1376+
{".pack", 0},
1377+
{".keep", 0},
1378+
{".rev", 0},
1379+
{".idx", 1}, /* The index file must be atomically moved last. */
1380+
};
1381+
1382+
trace2_region_enter("maintenance", "migrate_pack", the_repository);
1383+
1384+
basename = xstrndup(pack_filename, strlen(pack_filename) - 5 /*.pack*/);
1385+
1386+
/* A pack without an index file is not yet ready to be migrated. */
1387+
strbuf_addf(&src, "%s/%s%s", srcdir, basename, ".idx");
1388+
if (stat(src.buf, &st))
1389+
goto cleanup;
1390+
1391+
/* Move or copy files from the source directory to the destination. */
1392+
for (size_t i = 0; i < ARRAY_SIZE(files); i++) {
1393+
strbuf_reset(&src);
1394+
strbuf_addf(&src, "%s/%s%s", srcdir, basename, files[i].ext);
1395+
1396+
if (stat(src.buf, &st))
1397+
continue;
1398+
1399+
strbuf_reset(&dst);
1400+
strbuf_addf(&dst, "%s/%s%s", dstdir, basename, files[i].ext);
1401+
1402+
if (files[i].move) {
1403+
if (rename(src.buf, dst.buf))
1404+
die_errno(_("failed to move '%s' to '%s'"),
1405+
src.buf, dst.buf);
1406+
} else {
1407+
link_or_copy_or_die(src.buf, dst.buf);
1408+
}
1409+
}
1410+
1411+
/*
1412+
* Now the pack and all associated files exist at the destination we can
1413+
* now clean up the files in the source directory.
1414+
*/
1415+
for (size_t i = 0; i < ARRAY_SIZE(files); i++) {
1416+
/* Files that were moved rather than copied have no clean up. */
1417+
if (files[i].move)
1418+
continue;
1419+
1420+
strbuf_reset(&src);
1421+
strbuf_addf(&src, "%s/%s%s", srcdir, basename, files[i].ext);
1422+
if (unlink(src.buf))
1423+
warning_errno(_("failed to delete '%s'"), src.buf);
1424+
}
1425+
1426+
cleanup:
1427+
free(basename);
1428+
strbuf_release(&src);
1429+
strbuf_release(&dst);
1430+
1431+
trace2_region_leave("maintenance", "migrate_pack", the_repository);
1432+
}
1433+
1434+
static void move_pack_to_vfs_cache(const char *full_path, size_t full_path_len,
1435+
const char *file_name, UNUSED void *data)
1436+
{
1437+
char *srcdir;
1438+
struct strbuf dstdir = STRBUF_INIT;
1439+
1440+
/* We only care about the actual pack files here.
1441+
* The associated .idx, .keep, .rev files will be copied in tandem
1442+
* with the pack file, with the index file being moved last.
1443+
* The original locations of the non-index files will only deleted
1444+
* once all other files have been copied/moved.
1445+
*/
1446+
if (!ends_with(file_name, ".pack"))
1447+
return;
1448+
1449+
srcdir = xstrndup(full_path, full_path_len - strlen(file_name) - 1);
1450+
1451+
/* No cache or same source + desintation means there's no work to do. */
1452+
if (!vfs_object_dir || !fspathcmp(srcdir, vfs_object_dir))
1453+
goto cleanup;
1454+
1455+
strbuf_addf(&dstdir, "%s/pack", vfs_object_dir);
1456+
1457+
migrate_pack(srcdir, dstdir.buf, file_name);
1458+
1459+
cleanup:
1460+
free(srcdir);
1461+
strbuf_release(&dstdir);
1462+
}
1463+
1464+
static int maintenance_task_vfs_cache_move(UNUSED struct maintenance_run_opts *opts,
1465+
UNUSED struct gc_config *cfg)
1466+
{
1467+
struct repository *r = the_repository;
1468+
1469+
for_each_file_in_pack_dir(r->objects->odb->path, move_pack_to_vfs_cache,
1470+
NULL);
1471+
1472+
return 0;
1473+
}
1474+
13501475
typedef int maintenance_task_fn(struct maintenance_run_opts *opts,
13511476
struct gc_config *cfg);
13521477

@@ -1376,6 +1501,7 @@ enum maintenance_task_label {
13761501
TASK_GC,
13771502
TASK_COMMIT_GRAPH,
13781503
TASK_PACK_REFS,
1504+
TASK_VFS_CACHE_MOVE,
13791505

13801506
/* Leave as final value */
13811507
TASK__COUNT
@@ -1412,6 +1538,10 @@ static struct maintenance_task tasks[] = {
14121538
maintenance_task_pack_refs,
14131539
pack_refs_condition,
14141540
},
1541+
[TASK_VFS_CACHE_MOVE] = {
1542+
"vfs-cache-move",
1543+
maintenance_task_vfs_cache_move,
1544+
},
14151545
};
14161546

14171547
static int compare_tasks_by_selection(const void *a_, const void *b_)
@@ -1506,6 +1636,8 @@ static void initialize_maintenance_strategy(void)
15061636
tasks[TASK_LOOSE_OBJECTS].schedule = SCHEDULE_DAILY;
15071637
tasks[TASK_PACK_REFS].enabled = 1;
15081638
tasks[TASK_PACK_REFS].schedule = SCHEDULE_WEEKLY;
1639+
tasks[TASK_VFS_CACHE_MOVE].enabled = 1;
1640+
tasks[TASK_VFS_CACHE_MOVE].schedule = SCHEDULE_WEEKLY;
15091641
}
15101642
}
15111643

‎t/t7900-maintenance.sh‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,4 +1012,129 @@ test_expect_success 'repacking loose objects is quiet' '
10121012
)
10131013
'
10141014

1015+
test_expect_success 'vfs-cache-move task with no shared cache no op' '
1016+
test_when_finished "rm -rf repo" &&
1017+
git init repo &&
1018+
(
1019+
cd repo &&
1020+
1021+
test_commit something &&
1022+
git config set maintenance.gc.enabled false &&
1023+
git config set maintenance.vfs-cache-move.enabled true &&
1024+
git config set maintenance.vfs-cache-move.auto 1 &&
1025+
1026+
printf "blob\ndata <<END\n%s\nEND\n\n" 1 2 3 4 5 | \
1027+
git -c fastimport.unpackLimit=0 fast-import &&
1028+
find .git/objects/pack \
1029+
-type f \
1030+
\( -name "*.pack" \
1031+
-o -name "*.idx" \
1032+
-o -name "*.keep" \
1033+
-o -name "*.rev" \) >files.txt &&
1034+
1035+
git maintenance run &&
1036+
while IFS= read -r f; do
1037+
test_path_exists $f || exit 1
1038+
done <files.txt
1039+
)
1040+
'
1041+
1042+
test_expect_success 'vfs-cache-move task cache path same as pack dir no op' '
1043+
test_when_finished "rm -rf repo" &&
1044+
git init repo &&
1045+
(
1046+
cd repo &&
1047+
1048+
test_commit something &&
1049+
git config set gvfs.sharedcache .git/objects/pack &&
1050+
git config set maintenance.gc.enabled false &&
1051+
git config set maintenance.vfs-cache-move.enabled true &&
1052+
git config set maintenance.vfs-cache-move.auto 1 &&
1053+
1054+
printf "blob\ndata <<END\n%s\nEND\n\n" 1 2 3 4 5 | \
1055+
git -c fastimport.unpackLimit=0 fast-import &&
1056+
find .git/objects/pack \
1057+
-type f \
1058+
\( -name "*.pack" \
1059+
-o -name "*.idx" \
1060+
-o -name "*.keep" \
1061+
-o -name "*.rev" \) >files.txt &&
1062+
1063+
git maintenance run &&
1064+
while IFS= read -r f; do
1065+
test_path_exists $f || exit 1
1066+
done <files.txt
1067+
)
1068+
'
1069+
1070+
test_expect_success 'vfs-cache-move task no .rev or .keep' '
1071+
test_when_finished "rm -rf repo cache" &&
1072+
mkdir -p cache/pack &&
1073+
git init repo &&
1074+
(
1075+
cd repo &&
1076+
1077+
test_commit something &&
1078+
git config set gvfs.sharedcache ../cache &&
1079+
git config set maintenance.gc.enabled false &&
1080+
git config set maintenance.vfs-cache-move.enabled true &&
1081+
git config set maintenance.vfs-cache-move.auto 1 &&
1082+
1083+
printf "blob\ndata <<END\n%s\nEND\n\n" 1 2 3 4 5 | \
1084+
git -c fastimport.unpackLimit=0 fast-import &&
1085+
find .git/objects/pack \
1086+
-type f \
1087+
\( -name "*.pack" \
1088+
-o -name "*.idx" \) >src.txt &&
1089+
1090+
rm -f .git/objects/pack/*.rev .git/objects/pack/*.keep &&
1091+
1092+
sed "s|.*/|../cache/pack/|" src.txt >dst.txt &&
1093+
1094+
git maintenance run &&
1095+
while IFS= read -r f; do
1096+
test_path_is_missing $f || exit 1
1097+
done <src.txt &&
1098+
1099+
while IFS= read -r f; do
1100+
test_path_exists $f || exit 1
1101+
done <dst.txt
1102+
)
1103+
'
1104+
1105+
test_expect_success 'vfs-cache-move task success' '
1106+
test_when_finished "rm -rf repo cache" &&
1107+
mkdir -p cache/pack &&
1108+
git init repo &&
1109+
(
1110+
cd repo &&
1111+
1112+
test_commit something &&
1113+
git config set gvfs.sharedcache ../cache &&
1114+
git config set maintenance.gc.enabled false &&
1115+
git config set maintenance.vfs-cache-move.enabled true &&
1116+
git config set maintenance.vfs-cache-move.auto 1 &&
1117+
1118+
printf "blob\ndata <<END\n%s\nEND\n\n" 1 2 3 4 5 | \
1119+
git -c fastimport.unpackLimit=0 fast-import &&
1120+
find .git/objects/pack \
1121+
-type f \
1122+
\( -name "*.pack" \
1123+
-o -name "*.idx" \
1124+
-o -name "*.keep" \
1125+
-o -name "*.rev" \) >src.txt &&
1126+
1127+
sed "s|.*/|../cache/pack/|" src.txt >dst.txt &&
1128+
1129+
git maintenance run &&
1130+
while IFS= read -r f; do
1131+
test_path_is_missing $f || exit 1
1132+
done <src.txt &&
1133+
1134+
while IFS= read -r f; do
1135+
test_path_exists $f || exit 1
1136+
done <dst.txt
1137+
)
1138+
'
1139+
10151140
test_done

0 commit comments

Comments
 (0)