Skip to content

Commit ad1b4cf

Browse files
mosuemmosuem
committed
Add datagen support
1 parent 842c967 commit ad1b4cf

File tree

3 files changed

+212
-68
lines changed

3 files changed

+212
-68
lines changed

‎pkgs/intl4x/hook/build.dart

Lines changed: 188 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -34,30 +34,12 @@ Unknown build mode for icu4x. Set the `ICU4X_BUILD_MODE` environment variable wi
3434
'''),
3535
};
3636

37-
final builtLibrary = await buildMode.build();
37+
final buildResult = await buildMode.build();
3838
// For debugging purposes
3939
// ignore: deprecated_member_use
4040
output.addMetadatum(env, environmentBuildMode ?? 'fetch');
41-
42-
final nativeCodeAsset = NativeCodeAsset(
43-
package: package,
44-
name: assetId,
45-
linkMode: DynamicLoadingBundled(),
46-
architecture: config.targetArchitecture,
47-
os: config.targetOS,
48-
file: builtLibrary,
49-
);
50-
output.addAsset(
51-
nativeCodeAsset,
52-
linkInPackage: config.linkingEnabled ? config.packageName : null,
53-
);
54-
55-
output.addDependencies(
56-
[
57-
...buildMode.dependencies,
58-
config.packageRoot.resolve('hook/build.dart'),
59-
],
60-
);
41+
buildResult.addAssets(config, output);
42+
output.addDependencies(buildMode.dependencies);
6143
});
6244
}
6345

@@ -68,26 +50,74 @@ sealed class BuildMode {
6850

6951
List<Uri> get dependencies;
7052

71-
Future<Uri> build();
53+
Future<BuildResult> build();
54+
}
55+
56+
final class BuildResult {
57+
final Uri library;
58+
final Uri? datagen;
59+
final Uri? postcard;
60+
61+
BuildResult({
62+
required this.library,
63+
required this.datagen,
64+
required this.postcard,
65+
});
66+
67+
void addAssets(BuildConfig config, BuildOutput output) {
68+
output.addAssets(
69+
[
70+
NativeCodeAsset(
71+
package: package,
72+
name: assetId,
73+
linkMode: DynamicLoadingBundled(),
74+
architecture: config.targetArchitecture,
75+
os: config.targetOS,
76+
file: library,
77+
),
78+
if (datagen != null)
79+
DataAsset(
80+
package: package,
81+
name: 'datagen',
82+
file: datagen!,
83+
),
84+
if (postcard != null)
85+
DataAsset(
86+
package: package,
87+
name: 'postcard',
88+
file: postcard!,
89+
),
90+
],
91+
linkInPackage: config.linkingEnabled ? config.packageName : null,
92+
);
93+
}
7294
}
7395

7496
final class FetchMode extends BuildMode {
7597
FetchMode(super.config);
98+
final httpClient = HttpClient();
7699

77100
@override
78-
Future<Uri> build() async {
101+
Future<BuildResult> build() async {
79102
final target = '${config.targetOS}_${config.targetArchitecture}';
80-
final uri = Uri.parse(
103+
final dylibRemoteUri = Uri.parse(
81104
'https://github.com/dart-lang/i18n/releases/download/$version/$target');
82-
final request = await HttpClient().getUrl(uri);
83-
final response = await request.close();
84-
if (response.statusCode != 200) {
85-
throw ArgumentError('The request to $uri failed');
86-
}
87-
final dynamicLibrary =
88-
File.fromUri(config.outputDirectory.resolve(config.filename('icu4x')));
89-
await dynamicLibrary.create();
90-
await response.pipe(dynamicLibrary.openWrite());
105+
final dynamicLibrary = await fetchToFile(
106+
dylibRemoteUri,
107+
config.outputDirectory.resolve(config.filename('icu4x')),
108+
);
109+
110+
final datagen = await fetchToFile(
111+
Uri.parse(
112+
'https://github.com/dart-lang/i18n/releases/download/$version/$target-datagen'),
113+
config.outputDirectory.resolve('datagen'),
114+
);
115+
116+
final postcard = await fetchToFile(
117+
Uri.parse(
118+
'https://github.com/dart-lang/i18n/releases/download/$version/full.postcard'),
119+
config.outputDirectory.resolve('full.postcard'),
120+
);
91121

92122
final bytes = await dynamicLibrary.readAsBytes();
93123
final fileHash = sha256.convert(bytes).toString();
@@ -96,47 +126,90 @@ final class FetchMode extends BuildMode {
96126
config.targetArchitecture,
97127
)];
98128
if (fileHash == expectedFileHash) {
99-
return dynamicLibrary.uri;
129+
return BuildResult(
130+
library: dynamicLibrary.uri,
131+
datagen: datagen.uri,
132+
postcard: postcard.uri,
133+
);
100134
} else {
101135
throw Exception(
102-
'The pre-built binary for the target $target at $uri has a hash of '
103-
'$fileHash, which does not match $expectedFileHash fixed in the '
104-
'build hook of package:intl4x.');
136+
'The pre-built binary for the target $target at $dylibRemoteUri has a'
137+
' hash of $fileHash, which does not match $expectedFileHash fixed in'
138+
' the build hook of package:intl4x.');
105139
}
106140
}
107141

142+
Future<File> fetchToFile(Uri uri, Uri fileUri) async {
143+
final request = await httpClient.getUrl(uri);
144+
final response = await request.close();
145+
if (response.statusCode != 200) {
146+
throw ArgumentError('The request to $uri failed');
147+
}
148+
final file = File.fromUri(fileUri);
149+
await file.create();
150+
await response.pipe(file.openWrite());
151+
return file;
152+
}
153+
108154
@override
109155
List<Uri> get dependencies => [];
110156
}
111157

112158
final class LocalMode extends BuildMode {
113-
LocalMode(super.config);
114-
115-
String get _localBinaryPath {
116-
final localPath = Platform.environment['LOCAL_ICU4X_BINARY'];
117-
if (localPath != null) {
159+
final String localLibraryPath;
160+
final String? localDatagenPath;
161+
final String? localPostcardPath;
162+
163+
LocalMode(super.config)
164+
: localLibraryPath = _getFromEnvironment(
165+
'LOCAL_ICU4X_BINARY_${config.linkingEnabled ? 'STATIC' : 'DYNAMIC'}',
166+
true,
167+
)!,
168+
localDatagenPath = _getFromEnvironment('LOCAL_ICU4X_DATAGEN', false),
169+
localPostcardPath = _getFromEnvironment('LOCAL_ICU4X_POSTCARD', false);
170+
171+
static String? _getFromEnvironment(String key, bool mustExist) {
172+
final localPath = Platform.environment[key];
173+
if (localPath != null || !mustExist) {
118174
return localPath;
119175
}
120-
throw ArgumentError('`LOCAL_ICU4X_BINARY` is empty. '
176+
throw ArgumentError('`$key` is empty. '
121177
'If the `ICU4X_BUILD_MODE` is set to `local`, the '
122-
'`LOCAL_ICU4X_BINARY` environment variable must contain the path to '
123-
'the binary.');
178+
'`$key` environment variable must be set.');
124179
}
125180

126181
@override
127-
Future<Uri> build() async {
128-
final libFileName = config.filename('icu4x');
129-
final libFileUri = config.outputDirectory.resolve(libFileName);
130-
final file = File(_localBinaryPath);
131-
if (!(await file.exists())) {
132-
throw FileSystemException('Could not find binary.', _localBinaryPath);
182+
Future<BuildResult> build() async {
183+
final libFileUri = config.outputDirectory.resolve(config.filename('icu4x'));
184+
await copyFile(localLibraryPath, libFileUri);
185+
186+
final Uri? datagenFileUri;
187+
if (localDatagenPath != null) {
188+
datagenFileUri = config.outputDirectory.resolve('datagen');
189+
await copyFile(localDatagenPath!, datagenFileUri);
190+
} else {
191+
datagenFileUri = null;
192+
}
193+
194+
final Uri? postcardFileUri;
195+
if (localPostcardPath != null) {
196+
postcardFileUri = config.outputDirectory.resolve('postcard');
197+
await copyFile(localPostcardPath!, postcardFileUri);
198+
} else {
199+
postcardFileUri = null;
133200
}
134-
await file.copy(libFileUri.toFilePath(windows: Platform.isWindows));
135-
return libFileUri;
201+
202+
return BuildResult(
203+
library: libFileUri,
204+
datagen: datagenFileUri,
205+
postcard: postcardFileUri,
206+
);
136207
}
137208

138209
@override
139-
List<Uri> get dependencies => [Uri.file(_localBinaryPath)];
210+
List<Uri> get dependencies => [
211+
Uri.file(localLibraryPath),
212+
];
140213
}
141214

142215
final class CheckoutMode extends BuildMode {
@@ -145,7 +218,7 @@ final class CheckoutMode extends BuildMode {
145218
String? get workingDirectory => Platform.environment['LOCAL_ICU4X_CHECKOUT'];
146219

147220
@override
148-
Future<Uri> build() async {
221+
Future<BuildResult> build() async {
149222
if (workingDirectory == null) {
150223
throw ArgumentError('Specify the ICU4X checkout folder'
151224
'with the LOCAL_ICU4X_CHECKOUT variable');
@@ -159,11 +232,14 @@ final class CheckoutMode extends BuildMode {
159232
];
160233
}
161234

162-
Future<Uri> buildLib(BuildConfig config, String workingDirectory) async {
235+
Future<BuildResult> buildLib(
236+
BuildConfig config, String workingDirectory) async {
163237
final crateNameFixed = crateName.replaceAll('-', '_');
164238
final libFileName = config.filename(crateNameFixed);
165239

166240
final libFileUri = config.outputDirectory.resolve(libFileName);
241+
final datagenFileUri = config.outputDirectory.resolve('datagen');
242+
final postcardFileUri = config.outputDirectory.resolve('postcard');
167243
if (!config.dryRun) {
168244
final rustTarget = _asRustTarget(
169245
config.targetOS,
@@ -247,13 +323,56 @@ Future<Uri> buildLib(BuildConfig config, String workingDirectory) async {
247323
'release',
248324
libFileName,
249325
);
250-
final file = File(builtPath);
251-
if (!(await file.exists())) {
252-
throw FileSystemException('Building the dylib failed', builtPath);
326+
await copyFile(builtPath, libFileUri);
327+
328+
if (config.linkingEnabled) {
329+
final postcardPath = path.join(tempDir.path, 'full.postcard');
330+
await Process.run(
331+
'cargo',
332+
[
333+
'run',
334+
...['-p', 'icu_datagen'],
335+
'--',
336+
...['--locales', 'full'],
337+
...['--keys', 'all'],
338+
...['--format', 'blob'],
339+
...['--out', postcardPath],
340+
],
341+
workingDirectory: workingDirectory,
342+
);
343+
await copyFile(postcardPath, postcardFileUri);
344+
345+
final datagenPath = path.join(tempDir.path, 'datagen');
346+
final datagenDirectory = path.join(workingDirectory, 'provider/datagen');
347+
await Process.run(
348+
'rustup',
349+
['target', 'add', 'aarch64-unknown-linux-gnu'],
350+
workingDirectory: datagenDirectory,
351+
);
352+
await Process.run(
353+
'cargo',
354+
[
355+
'build',
356+
'--release',
357+
'--bin',
358+
'icu4x-datagen',
359+
'--no-default-features',
360+
...[
361+
'--features',
362+
'bin,blob_exporter,blob_input,rayon,experimental_components'
363+
],
364+
...['--target', 'aarch64-unknown-linux-gnu']
365+
],
366+
workingDirectory: datagenDirectory,
367+
);
368+
await copyFile(datagenPath, datagenFileUri);
253369
}
254-
await file.copy(libFileUri.toFilePath(windows: Platform.isWindows));
255370
}
256-
return libFileUri;
371+
return BuildResult(
372+
library: libFileUri,
373+
datagen: config.linkingEnabled ? datagenFileUri : null,
374+
postcard: config.linkingEnabled ? postcardFileUri : null,
375+
);
257376
}
258377

259378
String _asRustTarget(OS os, Architecture? architecture, bool isSimulator) {
@@ -297,3 +416,11 @@ extension on BuildConfig {
297416
String Function(String) get filename =>
298417
buildStatic ? targetOS.staticlibFileName : targetOS.dylibFileName;
299418
}
419+
420+
Future<void> copyFile(String path, Uri libFileUri) async {
421+
final file = File(path);
422+
if (!(await file.exists())) {
423+
throw FileSystemException('File does not exist.', path);
424+
}
425+
await file.copy(libFileUri.toFilePath(windows: Platform.isWindows));
426+
}

‎pkgs/intl4x/hook/link.dart

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import 'package:logging/logging.dart';
88
import 'package:native_assets_cli/native_assets_cli.dart';
99
import 'package:native_toolchain_c/native_toolchain_c.dart';
1010

11+
import 'locales.dart';
1112
import 'shared.dart';
1213

1314
// TODO(mosuem): Use `record_use` to automagically get the used symbols.
@@ -37,9 +38,8 @@ void main(List<String> arguments) {
3738
link(
3839
arguments,
3940
(config, output) async {
40-
final staticLib = config.assets.firstWhereOrNull(
41-
(element) => element.id == 'package:$package/$assetId',
42-
);
41+
final staticLib = config.assets
42+
.firstWhereOrNull((asset) => asset.id == 'package:$package/$assetId');
4343
if (staticLib == null) {
4444
// No static lib built, so assume a dynamic one was already bundled.
4545
return;
@@ -60,14 +60,15 @@ void main(List<String> arguments) {
6060
..onRecord.listen((record) => print(record.message)),
6161
);
6262

63-
final postcard = config.assets.first;
64-
final datagenTool = config.assets.first;
63+
final postcard =
64+
config.assets.firstWhere((asset) => asset.id.endsWith('postcard'));
65+
final datagenTool =
66+
config.assets.firstWhere((asset) => asset.id.endsWith('datagen'));
6567
final dylib = output.assets.first;
66-
final locales = ['de_DE', 'zh_hant_TW'];
6768

6869
await Process.run(datagenTool.file!.toFilePath(), [
6970
'--locales',
70-
locales.join(','),
71+
(await _customLocales ?? locales).join(','),
7172
'--input',
7273
postcard.file!.toFilePath(),
7374
'--keys-for-bin',
@@ -76,3 +77,14 @@ void main(List<String> arguments) {
7677
},
7778
);
7879
}
80+
81+
Future<List<String>?> get _customLocales async {
82+
final localPath = Platform.environment['ICU4X_LOCALE_LIST'];
83+
if (localPath != null) {
84+
final file = File(localPath);
85+
if (await file.exists()) {
86+
return file.readAsLines();
87+
}
88+
}
89+
return null;
90+
}

‎pkgs/intl4x/hook/locales.dart

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
final locales = ['en_US'];

0 commit comments

Comments
 (0)