Skip to content

Commit b8bd825

Browse files
sigurdmsigurdm
authored
new command pub cache gc (#4684)
1 parent 5215714 commit b8bd825

File tree

11 files changed

+452
-14
lines changed

11 files changed

+452
-14
lines changed

‎lib/src/command/cache.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import '../command.dart';
66
import 'cache_add.dart';
77
import 'cache_clean.dart';
8+
import 'cache_gc.dart';
89
import 'cache_list.dart';
910
import 'cache_preload.dart';
1011
import 'cache_repair.dart';
@@ -24,5 +25,6 @@ class CacheCommand extends PubCommand {
2425
addSubcommand(CacheCleanCommand());
2526
addSubcommand(CacheRepairCommand());
2627
addSubcommand(CachePreloadCommand());
28+
addSubcommand(CacheGcCommand());
2729
}
2830
}

‎lib/src/command/cache_gc.dart‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:convert';
6+
import 'dart:io';
7+
8+
import 'package:path/path.dart' as p;
9+
10+
import '../command.dart';
11+
import '../command_runner.dart';
12+
import '../io.dart';
13+
import '../log.dart' as log;
14+
import '../package_config.dart';
15+
import '../utils.dart';
16+
17+
class CacheGcCommand extends PubCommand {
18+
@override
19+
String get name => 'gc';
20+
@override
21+
String get description => 'Prunes unused packages from the system cache.';
22+
@override
23+
bool get takesArguments => false;
24+
25+
final dontRemoveFilesOlderThan = const Duration(hours: 2);
26+
27+
CacheGcCommand() {
28+
argParser.addFlag(
29+
'force',
30+
abbr: 'f',
31+
help: 'Prune cache without confirmation',
32+
hideNegatedUsage: true,
33+
);
34+
argParser.addFlag(
35+
'collect-recent',
36+
help: 'Also delete recent files',
37+
hideNegatedUsage: true,
38+
);
39+
argParser.addFlag(
40+
'dry-run',
41+
help: 'Print list of files that would be deleted',
42+
hideNegatedUsage: true,
43+
);
44+
}
45+
46+
@override
47+
Future<void> runProtected() async {
48+
final dryRun = argResults.flag('dry-run');
49+
final activeRoots = cache.activeRoots();
50+
// All the `activeRoots` that we could read and parse a
51+
// .dart_tool/packageConfig.json from.
52+
final validActiveRoots = <String>[];
53+
// All the rootUri paths to cached packages included from
54+
// `validActiveRoots`.
55+
final paths = <String>{};
56+
for (final packageConfigPath in activeRoots) {
57+
late final PackageConfig packageConfig;
58+
try {
59+
packageConfig = PackageConfig.fromJson(
60+
json.decode(readTextFile(packageConfigPath)),
61+
);
62+
} on IOException catch (e) {
63+
// Failed to read file - probably got deleted.
64+
log.fine('Failed to read packageConfig $packageConfigPath: $e');
65+
continue;
66+
} on FormatException catch (e) {
67+
log.warning(
68+
'Failed to decode packageConfig $packageConfigPath: $e.\n'
69+
'It could be corrupted',
70+
);
71+
// Failed to decode - probably corrupted.
72+
continue;
73+
}
74+
for (final package in packageConfig.packages) {
75+
final rootUri = p.canonicalize(
76+
package.resolvedRootDir(packageConfigPath),
77+
);
78+
if (p.isWithin(cache.rootDir, rootUri)) {
79+
paths.add(rootUri);
80+
}
81+
}
82+
validActiveRoots.add(packageConfigPath);
83+
}
84+
final now = DateTime.now();
85+
final allPathsToGC =
86+
[
87+
for (final source in cache.cachedSources)
88+
...await source.entriesToGc(
89+
cache,
90+
paths
91+
.where(
92+
(path) => p.isWithin(
93+
p.canonicalize(cache.rootDirForSource(source)),
94+
path,
95+
),
96+
)
97+
.toSet(),
98+
),
99+
].where((path) {
100+
// Only clear cache entries older than 2 hours to avoid race
101+
// conditions with ongoing `pub get` processes.
102+
final s = statPath(path);
103+
if (s.type == FileSystemEntityType.notFound) return false;
104+
if (argResults.flag('collect-recent')) return true;
105+
return now.difference(s.modified) > dontRemoveFilesOlderThan;
106+
}).toList();
107+
if (validActiveRoots.isEmpty) {
108+
log.message('Found no active projects.');
109+
} else {
110+
final s = validActiveRoots.length == 1 ? '' : 's';
111+
log.message('Found ${validActiveRoots.length} active project$s:');
112+
for (final packageConfigPath in validActiveRoots) {
113+
final parts = p.split(packageConfigPath);
114+
var projectDir = packageConfigPath;
115+
if (parts[parts.length - 2] == '.dart_tool' &&
116+
parts[parts.length - 1] == 'package_config.json') {
117+
projectDir = p.joinAll(parts.sublist(0, parts.length - 2));
118+
}
119+
log.message('* $projectDir');
120+
}
121+
}
122+
var sum = 0;
123+
for (final entry in allPathsToGC) {
124+
if (dirExists(entry)) {
125+
for (final file in listDir(
126+
entry,
127+
recursive: true,
128+
includeHidden: true,
129+
includeDirs: false,
130+
)) {
131+
sum += tryStatFile(file)?.size ?? 0;
132+
}
133+
} else {
134+
sum += tryStatFile(entry)?.size ?? 0;
135+
}
136+
}
137+
if (sum == 0) {
138+
log.message('No unused cache entries found.');
139+
return;
140+
}
141+
log.message('');
142+
log.message(
143+
'''
144+
All other projects ${dryRun ? 'would' : 'will'} need to run `$topLevelProgram pub get` again to work correctly.''',
145+
);
146+
log.message(
147+
'${dryRun ? 'Would' : 'Will'} recover ${readableFileSize(sum)}.',
148+
);
149+
if (dryRun) {
150+
log.message('Would delete:');
151+
for (final path in allPathsToGC..sort()) {
152+
log.message(path);
153+
}
154+
} else if (argResults.flag('force') ||
155+
await confirm('Are you sure you want to continue?')) {
156+
await log.progress('Deleting unused cache entries', () async {
157+
for (final path in allPathsToGC..sort()) {
158+
tryDeleteEntry(path);
159+
}
160+
});
161+
}
162+
}
163+
}

‎lib/src/command/lish.dart‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ the \$PUB_HOSTED_URL environment variable.''');
368368
baseDir: entrypoint.workPackage.dir,
369369
).toBytes();
370370

371-
final size = _readableFileSize(packageBytes.length);
371+
final size = readableFileSize(packageBytes.length);
372372
log.message('\nTotal compressed archive size: $size.\n');
373373

374374
final validationResult =
@@ -536,18 +536,6 @@ the \$PUB_HOSTED_URL environment variable.''');
536536
}
537537
}
538538

539-
String _readableFileSize(int size) {
540-
if (size >= 1 << 30) {
541-
return '${size ~/ (1 << 30)} GB';
542-
} else if (size >= 1 << 20) {
543-
return '${size ~/ (1 << 20)} MB';
544-
} else if (size >= 1 << 10) {
545-
return '${size ~/ (1 << 10)} KB';
546-
} else {
547-
return '<1 KB';
548-
}
549-
}
550-
551539
class _Publication {
552540
Uint8List packageBytes;
553541
int warningCount;

‎lib/src/source/cached.dart‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,14 @@ abstract class CachedSource extends Source {
7676
/// Returns a list of results indicating for each if that package was
7777
/// successfully repaired.
7878
Future<Iterable<RepairResult>> repairCachedPackages(SystemCache cache);
79+
80+
/// Return all directories inside this source that can be removed while
81+
/// preserving the packages given by [alivePackages] a list of package root
82+
/// directories. They should all be canonicalized.
83+
Future<List<String>> entriesToGc(
84+
SystemCache cache,
85+
Set<String> alivePackages,
86+
);
7987
}
8088

8189
/// The result of repairing a single cache entry.

‎lib/src/source/git.dart‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -900,6 +900,74 @@ class GitSource extends CachedSource {
900900
}
901901
return name;
902902
}
903+
904+
@override
905+
Future<List<String>> entriesToGc(
906+
SystemCache cache,
907+
Set<String> alivePackages,
908+
) async {
909+
final rootDir = p.canonicalize(cache.rootDirForSource(this));
910+
if (!entryExists(rootDir)) return const [];
911+
912+
final gitDirsToRemove = <String>{};
913+
// First enumerate all git repos inside [rootDir].
914+
for (final entry in listDir(rootDir)) {
915+
final gitEntry = p.join(entry, '.git');
916+
if (!entryExists(gitEntry)) continue;
917+
gitDirsToRemove.add(p.canonicalize(entry));
918+
}
919+
final cacheDirsToRemove = <String>{};
920+
try {
921+
cacheDirsToRemove.addAll(
922+
listDir(p.join(rootDir, 'cache')).map(p.canonicalize),
923+
);
924+
} on IOException {
925+
// Most likely the directory didn't exist.
926+
// ignore.
927+
}
928+
// For each package walk up parent directories to find the containing git
929+
// repo, and mark it alive by removing from `gitDirsToRemove`.
930+
for (final alivePackage in alivePackages) {
931+
var candidate = p.canonicalize(alivePackage);
932+
while (!p.equals(candidate, rootDir)) {
933+
if (gitDirsToRemove.remove(candidate)) {
934+
// Package is alive, now also retain its cachedir.
935+
//
936+
// TODO(sigurdm): Should we just GC all cache-dirs? They are not
937+
// needed for consuming packages, and most likely will be recreated
938+
// when needed.
939+
final gitEntry = p.join(candidate, '.git');
940+
try {
941+
if (dirExists(gitEntry)) {
942+
final path =
943+
(await git.run([
944+
'remote',
945+
'get-url',
946+
'origin',
947+
], workingDir: candidate)).split('\n').first;
948+
cacheDirsToRemove.remove(p.canonicalize(path));
949+
} else if (fileExists(gitEntry)) {
950+
// Potential future - using worktrees.
951+
final path =
952+
(await git.run([
953+
'worktree',
954+
'list',
955+
'--porcelain',
956+
], workingDir: candidate)).split('\n').first.split(' ').last;
957+
cacheDirsToRemove.remove(p.canonicalize(path));
958+
}
959+
} on git.GitException catch (e) {
960+
log.fine('Failed to find canonical cache for $candidate, $e');
961+
}
962+
break;
963+
}
964+
// Try the parent directory
965+
candidate = p.dirname(candidate);
966+
}
967+
}
968+
969+
return [...cacheDirsToRemove, ...gitDirsToRemove];
970+
}
903971
}
904972

905973
class GitDescription extends Description {

‎lib/src/source/hosted.dart‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1765,6 +1765,62 @@ See $contentHashesDocumentationUrl.
17651765
}
17661766
}
17671767

1768+
@override
1769+
Future<List<String>> entriesToGc(
1770+
SystemCache cache,
1771+
Set<String> alivePackages,
1772+
) async {
1773+
final root = p.canonicalize(cache.rootDirForSource(this));
1774+
final result = <String>{};
1775+
final List<String> hostDirs;
1776+
1777+
try {
1778+
hostDirs = listDir(root);
1779+
} on IOException {
1780+
// Hosted cache seems uninitialized. GC nothing.
1781+
return [];
1782+
}
1783+
for (final hostDir in hostDirs) {
1784+
final List<String> packageDirs;
1785+
try {
1786+
packageDirs = listDir(hostDir).map(p.canonicalize).toList();
1787+
} on IOException {
1788+
// Failed to list `hostDir`. Perhaps a stray file? Skip.
1789+
continue;
1790+
}
1791+
for (final packageDir in packageDirs) {
1792+
if (!alivePackages.contains(packageDir)) {
1793+
result.add(packageDir);
1794+
// Also clear the associated hash file.
1795+
final hashFile = p.join(
1796+
cache.rootDir,
1797+
'hosted-hashes',
1798+
p.basename(hostDir),
1799+
'${p.basename(packageDir)}.sha256',
1800+
);
1801+
if (fileExists(hashFile)) {
1802+
result.add(hashFile);
1803+
}
1804+
}
1805+
}
1806+
// Clear all version listings older than two days, they'd likely need to
1807+
// be re-fetched anyways:
1808+
for (final cacheFile in listDir(
1809+
p.join(hostDir, _versionListingDirectory),
1810+
)) {
1811+
final stat = tryStatFile(cacheFile);
1812+
1813+
if (stat != null &&
1814+
DateTime.now().difference(stat.modified) >
1815+
const Duration(days: 2)) {
1816+
result.add(cacheFile);
1817+
}
1818+
}
1819+
}
1820+
1821+
return result.toList();
1822+
}
1823+
17681824
/// Enables speculative prefetching of dependencies of packages queried with
17691825
/// [doGetVersions].
17701826
Future<T> withPrefetching<T>(Future<T> Function() callback) async {

‎lib/src/utils.dart‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -825,3 +825,15 @@ extension ExpectEntries on YamlList {
825825
),
826826
];
827827
}
828+
829+
String readableFileSize(int size) {
830+
if (size >= 1 << 30) {
831+
return '${size ~/ (1 << 30)} GB';
832+
} else if (size >= 1 << 20) {
833+
return '${size ~/ (1 << 20)} MB';
834+
} else if (size >= 1 << 10) {
835+
return '${size ~/ (1 << 10)} KB';
836+
} else {
837+
return '<1 KB';
838+
}
839+
}

‎pubspec.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ dependencies:
1919
http_parser: ^4.1.2
2020
meta: ^1.17.0
2121
path: ^1.9.1
22-
pool: ^1.5.1
22+
pool: ^1.0.0
2323
pub_semver: ^2.2.0
2424
shelf: ^1.4.2
2525
source_span: ^1.10.1

0 commit comments

Comments
 (0)