Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 4946d44

Browse files
jason-simmonsjason-simmons
authored
Skip license processing for top-level source directories that are unchanged (#3437)
See flutter/flutter#8106
1 parent 74de13c commit 4946d44

File tree

8 files changed

+52037
-49856
lines changed

8 files changed

+52037
-49856
lines changed

‎tools/licenses/lib/main.dart

Lines changed: 130 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,16 @@
44

55
// See README in this directory for information on how this code is organised.
66

7+
import 'dart:async';
78
import 'dart:collection';
89
import 'dart:convert';
910
import 'dart:io' as system;
1011
import 'dart:math' as math;
1112

13+
import 'package:args/args.dart';
14+
import 'package:crypto/crypto.dart' as crypto;
15+
import 'package:path/path.dart' as path;
16+
1217
import 'filesystem.dart' as fs;
1318
import 'licenses.dart';
1419
import 'patterns.dart';
@@ -919,6 +924,8 @@ class RepositoryDirectory extends RepositoryEntry implements LicenseSource {
919924
final List<RepositoryLicensedFile> _files = <RepositoryLicensedFile>[];
920925
final List<RepositoryLicenseFile> _licenses = <RepositoryLicenseFile>[];
921926

927+
List<RepositoryDirectory> get subdirectories => _subdirectories;
928+
922929
final Map<String, RepositoryEntry> _childrenByName = <String, RepositoryEntry>{};
923930

924931
// the bit at the beginning excludes files like "license.py".
@@ -1235,6 +1242,33 @@ class RepositoryDirectory extends RepositoryEntry implements LicenseSource {
12351242
result += directory.fileCount;
12361243
return result;
12371244
}
1245+
1246+
Iterable<RepositoryLicensedFile> get _allFiles sync* {
1247+
for (RepositoryLicensedFile file in _files) {
1248+
if (file.isIncludedInBuildProducts)
1249+
yield file;
1250+
}
1251+
for (RepositoryDirectory directory in _subdirectories) {
1252+
yield* directory._allFiles;
1253+
}
1254+
}
1255+
1256+
Stream<List<int>> _signatureStream(List files) async* {
1257+
for (RepositoryLicensedFile file in files) {
1258+
yield file.io.fullName.codeUnits;
1259+
yield file.io.readBytes();
1260+
}
1261+
}
1262+
1263+
/// Compute a signature representing a hash of all the licensed files within
1264+
/// this directory tree.
1265+
Future<String> get signature async {
1266+
List allFiles = _allFiles.toList();
1267+
allFiles.sort((RepositoryLicensedFile a, RepositoryLicensedFile b) =>
1268+
a.io.fullName.compareTo(b.io.fullName));
1269+
crypto.Digest digest = await crypto.md5.bind(_signatureStream(allFiles)).single;
1270+
return digest.bytes.map((int e) => e.toRadixString(16).padLeft(2, '0')).join();
1271+
}
12381272
}
12391273

12401274
class RepositoryGenericThirdPartyDirectory extends RepositoryDirectory {
@@ -2237,53 +2271,115 @@ class Progress {
22372271

22382272
// MAIN
22392273

2240-
void main(List<String> arguments) {
2241-
if (arguments.length != 1) {
2242-
print('Usage: dart lib/main.dart path/to/engine/root/src');
2274+
Future<Null> main(List<String> arguments) async {
2275+
final ArgParser parser = new ArgParser()
2276+
..addOption('src', help: 'The root of the engine source')
2277+
..addOption('out', help: 'The directory where output is written')
2278+
..addOption('golden', help: 'The directory containing golden results')
2279+
..addFlag('release', help: 'Print output in the format used for product releases');
2280+
2281+
ArgResults argResults = parser.parse(arguments);
2282+
bool releaseMode = argResults['release'];
2283+
if (argResults['src'] == null) {
2284+
print('Flutter license script: Must provide --src directory');
2285+
print(parser.usage);
22432286
system.exit(1);
22442287
}
2288+
if (!releaseMode) {
2289+
if (argResults['out'] == null || argResults['golden'] == null) {
2290+
print('Flutter license script: Must provide --out and --golden directories in non-release mode');
2291+
print(parser.usage);
2292+
system.exit(1);
2293+
}
2294+
if (!system.FileSystemEntity.isDirectorySync(argResults['golden'])) {
2295+
print('Flutter license script: Golden directory does not exist');
2296+
print(parser.usage);
2297+
system.exit(1);
2298+
}
2299+
system.Directory out = new system.Directory(argResults['out']);
2300+
if (!out.existsSync())
2301+
out.createSync(recursive: true);
2302+
}
22452303

22462304
try {
22472305
system.stderr.writeln('Finding files...');
2248-
final RepositoryDirectory root = new RepositoryRoot(new fs.FileSystemDirectory.fromPath(arguments.single));
2249-
system.stderr.writeln('Collecting licenses...');
2250-
Progress progress = new Progress(root.fileCount);
2251-
List<License> licenses = new Set<License>.from(root.getLicenses(progress).toList()).toList();
2252-
progress.label = 'Dumping results...';
2253-
bool done = false;
2254-
List<License> usedLicenses = licenses.where((License license) => license.isUsed).toList();
2255-
assert(() {
2256-
print('UNUSED LICENSES:\n');
2257-
List<String> unusedLicenses = licenses
2258-
.where((License license) => !license.isUsed)
2259-
.map((License license) => license.toString())
2260-
.toList();
2261-
unusedLicenses.sort();
2262-
print(unusedLicenses.join('\n\n'));
2263-
print('~' * 80);
2264-
print('USED LICENSES:\n');
2265-
List<String> output = usedLicenses.map((License license) => license.toString()).toList();
2266-
output.sort();
2267-
print(output.join('\n\n'));
2268-
done = true;
2269-
return true;
2270-
});
2271-
if (!done) {
2306+
final RepositoryDirectory root = new RepositoryRoot(new fs.FileSystemDirectory.fromPath(argResults['src']));
2307+
2308+
if (releaseMode) {
2309+
system.stderr.writeln('Collecting licenses...');
2310+
Progress progress = new Progress(root.fileCount);
2311+
List<License> licenses = new Set<License>.from(root.getLicenses(progress).toList()).toList();
22722312
if (progress.hadErrors)
22732313
throw 'Had failures while collecting licenses.';
2274-
List<String> output = usedLicenses
2314+
progress.label = 'Dumping results...';
2315+
List<String> output = licenses
2316+
.where((License license) => license.isUsed)
22752317
.map((License license) => license.toStringFormal())
22762318
.where((String text) => text != null)
22772319
.toList();
22782320
output.sort();
22792321
print(output.join('\n${"-" * 80}\n'));
2322+
} else {
2323+
RegExp signaturePattern = new RegExp(r'Signature: (\w+)');
2324+
2325+
for (RepositoryDirectory component in root.subdirectories) {
2326+
system.stderr.writeln('Collecting licenses for ${component.io.name}');
2327+
2328+
String signature;
2329+
if (component.io.name == 'flutter') {
2330+
// Always run the full license check on the flutter tree. This tree is
2331+
// relatively small but changes frequently in ways that do not affect
2332+
// the license output, and we don't want to require updates to the golden
2333+
// signature for those changes.
2334+
signature = null;
2335+
} else {
2336+
signature = await component.signature;
2337+
}
2338+
2339+
// Check whether the golden file matches the signature of the current contents
2340+
// of this directory.
2341+
system.File goldenFile = new system.File(
2342+
path.join(argResults['golden'], 'licenses_${component.io.name}'));
2343+
String goldenSignature = await goldenFile.openRead()
2344+
.transform(UTF8.decoder).transform(new LineSplitter()).first;
2345+
Match goldenMatch = signaturePattern.matchAsPrefix(goldenSignature);
2346+
if (goldenMatch != null && goldenMatch.group(1) == signature) {
2347+
system.stderr.writeln(' Skipping this component - no change in signature');
2348+
continue;
2349+
}
2350+
2351+
Progress progress = new Progress(component.fileCount);
2352+
2353+
system.File outFile = new system.File(
2354+
path.join(argResults['out'], 'licenses_${component.io.name}'));
2355+
system.IOSink sink = outFile.openWrite();
2356+
if (signature != null)
2357+
sink.writeln('Signature: $signature\n');
2358+
2359+
List<License> licenses = new Set<License>.from(
2360+
component.getLicenses(progress).toList()).toList();
2361+
2362+
sink.writeln('UNUSED LICENSES:\n');
2363+
List<String> unusedLicenses = licenses
2364+
.where((License license) => !license.isUsed)
2365+
.map((License license) => license.toString())
2366+
.toList();
2367+
unusedLicenses.sort();
2368+
sink.writeln(unusedLicenses.join('\n\n'));
2369+
sink.writeln('~' * 80);
2370+
2371+
sink.writeln('USED LICENSES:\n');
2372+
List<License> usedLicenses = licenses.where((License license) => license.isUsed).toList();
2373+
List<String> output = usedLicenses.map((License license) => license.toString()).toList();
2374+
output.sort();
2375+
sink.writeln(output.join('\n\n'));
2376+
sink.writeln('Total license count: ${licenses.length}');
2377+
2378+
await sink.close();
2379+
progress.label = 'Done.';
2380+
system.stderr.writeln('');
2381+
}
22802382
}
2281-
assert(() {
2282-
print('Total license count: ${licenses.length}');
2283-
progress.label = 'Done.';
2284-
print('$progress');
2285-
return true;
2286-
});
22872383
} catch (e, stack) {
22882384
system.stderr.writeln('failure: $e\n$stack');
22892385
system.stderr.writeln('aborted.');

‎tools/licenses/pubspec.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ dependencies:
33
path: ^1.3.0
44
archive: ^1.0.24
55
args: 0.13.7
6-
6+
crypto: ^2.0.1

‎travis/licenses.sh

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,23 @@
1+
shopt -s nullglob
2+
13
echo "Verifying license script is still happy..."
2-
(cd flutter/tools/licenses; pub get; dart --checked lib/main.dart ../../.. > ../../../out/license-script-output)
4+
(cd flutter/tools/licenses; pub get; dart --checked lib/main.dart --src ../../.. --out ../../../out/license_script_output --golden ../../travis/licenses_golden)
5+
6+
for f in out/license_script_output/licenses_*; do
7+
if ! cmp -s flutter/travis/licenses_golden/$(basename $f) $f
8+
then
9+
echo "License script got different results than expected for $f."
10+
echo "Please rerun the licenses script locally to verify that it is"
11+
echo "correctly catching any new licenses for anything you may have"
12+
echo "changed, and then update this file:"
13+
echo " flutter/sky/packages/sky_engine/LICENSE"
14+
echo "For more information, see the script in:"
15+
echo " https://github.com/flutter/engine/tree/master/tools/licenses"
16+
echo ""
17+
diff -U 6 flutter/travis/licenses_golden/$(basename $f) $f
18+
exit 1
19+
fi
20+
done
321

4-
if cmp -s flutter/travis/licenses.golden out/license-script-output
5-
then
6-
echo "Licenses are as expected."
7-
exit 0
8-
else
9-
echo "License script got different results than expected."
10-
echo "Please rerun the licenses script locally to verify that it is"
11-
echo "correctly catching any new licenses for anything you may have"
12-
echo "changed, and then update this file:"
13-
echo " flutter/sky/packages/sky_engine/LICENSE"
14-
echo "For more information, see the script in:"
15-
echo " https://github.com/flutter/engine/tree/master/tools/licenses"
16-
echo ""
17-
diff -U 6 flutter/travis/licenses.golden out/license-script-output
18-
exit 1
19-
fi
22+
echo "Licenses are as expected."
23+
exit 0

0 commit comments

Comments
 (0)