[native_assets] Use kernel concatenation (#147158)

Uses kernel concatenation to concatenate the `program.dill` and `native_assets.dill`.

This will enable compiling `native_assets.dill` after `program.dill` so that tree-shaking information can be used.

Issue:

* flutter/flutter#146270

## Implementation choices

* We can either run the frontend_server twice as a process invocation (current PR), or keep the process running after the dart compilation and then shut it down after native assets compilation to kernel. Keeping the process running would require more coordination between different `Target`s.
* We can either chose to only do separate kernel file compilation in release (AOT) builds, or do it both for JIT and AOT (current PR). Doing it differently in JIT/AOT means more conditionals in lib/src/build_system/targets/, doing it single-shot in JIT might be more performant though.

Update: Both of these are mitigated by not running the kernel compiler process if there are no native assets at all.

## Implementation notes

This only updates `flutter assemble`.

The kernel compilation calls in `flutter run` do not require kernel concatenation. The native-assets-kernel-mapping in `flutter run` contains results from `--dry-run` invocations of `hook/build.dart` (and in the future `hook/link.dart`). These `--dry-run` invocations do not get access to tree-shaking information, so the `link` hook to be introduced later can be run before Dart compilation.

## Tests

* Updated the `Target`s tests to reflect the new implementation.

## Related CLs

* frontend_server support: https://dart-review.googlesource.com/c/sdk/+/363567
* Same PR for Dart standalone: https://dart-review.googlesource.com/c/sdk/+/362384

Author: dcharkes

packages/flutter_tools/lib/src/build_system/targets/common.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'package:package_config/package_config.dart';
+import 'package:yaml/yaml.dart';
 
 import '../../artifacts.dart';
 import '../../base/build.dart';
@@ -121,15 +122,17 @@ class ReleaseCopyFlutterBundle extends CopyFlutterBundle {
 /// even though it is not listed as an input. Pub inserts a timestamp into
 /// the file which causes unnecessary rebuilds, so instead a subset of the contents
 /// are used an input instead.
-class KernelSnapshot extends Target {
-  const KernelSnapshot();
+///
+/// This kernel snapshot is concatenated with the [KernelSnapshotNativeAssets]
+/// inside [KernelSnapshot] byte-wise to create the combined kernel snapshot.
+class KernelSnapshotProgram extends Target {
+  const KernelSnapshotProgram();
 
   @override
-  String get name => 'kernel_snapshot';
+  String get name => 'kernel_snapshot_program';
 
   @override
   List<Source> get inputs => const <Source>[
-    Source.pattern('{BUILD_DIR}/native_assets.yaml'),
     Source.pattern('{PROJECT_DIR}/.dart_tool/package_config_subset'),
     Source.pattern('{FLUTTER_ROOT}/packages/flutter_tools/lib/src/build_system/targets/common.dart'),
     Source.artifact(Artifact.platformKernelDill),
@@ -139,7 +142,9 @@ class KernelSnapshot extends Target {
   ];
 
   @override
-  List<Source> get outputs => const <Source>[];
+  List<Source> get outputs => const <Source>[
+    // TODO(mosuem): Should output resources.json. https://github.com/flutter/flutter/issues/146263
+  ];
 
   @override
   List<String> get depfiles => <String>[
@@ -148,11 +153,12 @@ class KernelSnapshot extends Target {
 
   @override
   List<Target> get dependencies => const <Target>[
-    NativeAssets(),
     GenerateLocalizationsTarget(),
     DartPluginRegistrantTarget(),
   ];
 
+  static const String dillName = 'program.dill';
+
   @override
   Future<void> build(Environment environment) async {
     final KernelCompiler compiler = KernelCompiler(
@@ -186,13 +192,6 @@ class KernelSnapshot extends Target {
     final List<String>? fileSystemRoots = environment.defines[kFileSystemRoots]?.split(',');
     final String? fileSystemScheme = environment.defines[kFileSystemScheme];
 
-    final File nativeAssetsFile = environment.buildDir.childFile('native_assets.yaml');
-    final String nativeAssets = nativeAssetsFile.path;
-    if (!await nativeAssetsFile.exists()) {
-      throwToolExit("$nativeAssets doesn't exist.");
-    }
-    environment.logger.printTrace('Embedding native assets mapping $nativeAssets in kernel.');
-
     TargetModel targetModel = TargetModel.flutter;
     if (targetPlatform == TargetPlatform.fuchsia_x64 ||
         targetPlatform == TargetPlatform.fuchsia_arm64) {
@@ -242,6 +241,8 @@ class KernelSnapshot extends Target {
       logger: environment.logger,
     );
 
+    final String dillPath = environment.buildDir.childFile(dillName).path;
+
     final CompilerOutput? output = await compiler.compile(
       sdkRoot: environment.artifacts.getArtifactPath(
         Artifact.flutterPatchedSdkPath,
@@ -252,9 +253,8 @@ class KernelSnapshot extends Target {
       buildMode: buildMode,
       trackWidgetCreation: trackWidgetCreation && buildMode != BuildMode.release,
       targetModel: targetModel,
-      outputFilePath: environment.buildDir.childFile('app.dill').path,
-      initializeFromDill: buildMode.isPrecompiled ? null :
-          environment.buildDir.childFile('app.dill').path,
+      outputFilePath: dillPath,
+      initializeFromDill: buildMode.isPrecompiled ? null : dillPath,
       packagesPath: packagesFile.path,
       linkPlatformKernelIn: forceLinkPlatform || buildMode.isPrecompiled,
       mainPath: targetFileAbsolute,
@@ -268,6 +268,109 @@ class KernelSnapshot extends Target {
       buildDir: environment.buildDir,
       targetOS: targetOS,
       checkDartPluginRegistry: environment.generateDartPluginRegistry,
+    );
+    if (output == null || output.errorCount != 0) {
+      throw Exception();
+    }
+  }
+}
+
+/// Generate a kernel snapshot of the native assets mapping for resolving
+/// `@Native` assets at runtime.
+///
+/// This kernel snapshot is concatenated to the [KernelSnapshotProgram]
+/// inside [KernelSnapshot] to create the combined kernel snapshot.
+class KernelSnapshotNativeAssets extends Target {
+  const KernelSnapshotNativeAssets();
+
+  @override
+  String get name =>  'kernel_snapshot_native_assets';
+
+  @override
+  List<Source> get inputs => <Source>[
+    const Source.pattern('{BUILD_DIR}/native_assets.yaml'),
+    ...const KernelSnapshotProgram().inputs,
+  ];
+
+  @override
+  List<Source> get outputs => const <Source>[];
+
+  @override
+  List<String> get depfiles => const <String>[];
+
+  @override
+  List<Target> get dependencies => <Target>[
+    const NativeAssets(),
+  ];
+
+  static const String dillName = 'native_assets.dill';
+
+  @override
+  Future<void> build(Environment environment) async {
+    final File nativeAssetsFile = environment.buildDir.childFile('native_assets.yaml');
+    final File dillFile = environment.buildDir.childFile(dillName);
+
+    final YamlNode nativeAssetContents = loadYamlNode(await nativeAssetsFile.readAsString());
+    final Object? nativeAssetsInYaml = (nativeAssetContents as Map<Object?, Object?>)['native-assets'];
+    if (nativeAssetsInYaml is! Map || nativeAssetsInYaml.isEmpty) {
+      // Write an empty file to make concatenation a no-op.
+      // Write the file out to disk for caching.
+      await dillFile.writeAsBytes(<int>[]);
+      return;
+    }
+
+    final KernelCompiler compiler = KernelCompiler(
+      fileSystem: environment.fileSystem,
+      logger: environment.logger,
+      processManager: environment.processManager,
+      artifacts: environment.artifacts,
+      fileSystemRoots: <String>[],
+    );
+    final String? buildModeEnvironment = environment.defines[kBuildMode];
+    if (buildModeEnvironment == null) {
+      throw MissingDefineException(kBuildMode, 'kernel_snapshot');
+    }
+    final String? targetPlatformEnvironment = environment.defines[kTargetPlatform];
+    if (targetPlatformEnvironment == null) {
+      throw MissingDefineException(kTargetPlatform, 'kernel_snapshot');
+    }
+    final BuildMode buildMode = BuildMode.fromCliName(buildModeEnvironment);
+    final File packagesFile = environment.projectDir
+      .childDirectory('.dart_tool')
+      .childFile('package_config.json');
+
+    final TargetPlatform targetPlatform = getTargetPlatformForName(targetPlatformEnvironment);
+
+    final String? frontendServerStarterPath = environment.defines[kFrontendServerStarterPath];
+
+    final String nativeAssets = nativeAssetsFile.path;
+    if (!await nativeAssetsFile.exists()) {
+      throwToolExit("$nativeAssets doesn't exist.");
+    }
+    environment.logger.printTrace('Embedding native assets mapping $nativeAssets in kernel.');
+
+    final PackageConfig packageConfig = await loadPackageConfigWithLogging(
+      packagesFile,
+      logger: environment.logger,
+    );
+
+    final String dillPath = dillFile.path;
+
+    final CompilerOutput? output = await compiler.compile(
+      sdkRoot: environment.artifacts.getArtifactPath(
+        Artifact.flutterPatchedSdkPath,
+        platform: targetPlatform,
+        mode: buildMode,
+      ),
+      aot: buildMode.isPrecompiled,
+      buildMode: buildMode,
+      trackWidgetCreation: false,
+      outputFilePath: dillPath,
+      packagesPath: packagesFile.path,
+      frontendServerStarterPath: frontendServerStarterPath,
+      packageConfig: packageConfig,
+      buildDir: environment.buildDir,
+      dartDefines: <String>[],
       nativeAssets: nativeAssets,
     );
     if (output == null || output.errorCount != 0) {
@@ -276,6 +379,43 @@ class KernelSnapshot extends Target {
   }
 }
 
+class KernelSnapshot extends Target {
+  const KernelSnapshot();
+
+  @override
+  String get name => 'kernel_snapshot';
+
+  @override
+  List<Target> get dependencies => const <Target>[
+    KernelSnapshotProgram(),
+    KernelSnapshotNativeAssets(),
+  ];
+
+  @override
+  List<Source> get inputs => <Source>[];
+
+  @override
+  List<Source> get outputs => <Source>[];
+
+  static const String dillName = 'app.dill';
+
+  @override
+  Future<void> build(Environment environment) async {
+    final File programDill = environment.buildDir.childFile(
+      KernelSnapshotProgram.dillName,
+    );
+    final File nativeAssetsDill = environment.buildDir.childFile(
+      KernelSnapshotNativeAssets.dillName,
+    );
+    final File dill = environment.buildDir.childFile(dillName);
+    await programDill.copy(dill.path);
+    await dill.writeAsBytes(
+      await nativeAssetsDill.readAsBytes(),
+      mode: FileMode.append,
+    );
+  }
+}
+
 /// Supports compiling a dart kernel file to an ELF binary.
 abstract class AotElfBase extends Target {
   const AotElfBase();

packages/flutter_tools/lib/src/build_system/targets/native_assets.dart

@@ -371,13 +371,17 @@ class NativeAssets extends Target {
   ];
 
   @override
-  List<Target> get dependencies => <Target>[];
+  List<Target> get dependencies => const <Target>[
+    // In AOT, depends on tree-shaking information (resources.json) from compiling dart.
+    KernelSnapshotProgram(),
+  ];
 
   @override
   List<Source> get inputs => const <Source>[
     Source.pattern('{FLUTTER_ROOT}/packages/flutter_tools/lib/src/build_system/targets/native_assets.dart'),
     // If different packages are resolved, different native assets might need to be built.
     Source.pattern('{PROJECT_DIR}/.dart_tool/package_config_subset'),
+    // TODO(mosuem): Should consume resources.json. https://github.com/flutter/flutter/issues/146263
   ];
 
   @override

packages/flutter_tools/lib/src/compile.dart

@@ -247,12 +247,14 @@ class KernelCompiler {
       sdkRoot = '$sdkRoot/';
     }
     String? mainUri;
-    final File mainFile = _fileSystem.file(mainPath);
-    final Uri mainFileUri = mainFile.uri;
-    if (packagesPath != null) {
-      mainUri = packageConfig.toPackageUri(mainFileUri)?.toString();
+    if (mainPath != null) {
+      final File mainFile = _fileSystem.file(mainPath);
+      final Uri mainFileUri = mainFile.uri;
+      if (packagesPath != null) {
+        mainUri = packageConfig.toPackageUri(mainFileUri)?.toString();
+      }
+      mainUri ??= toMultiRootPath(mainFileUri, _fileSystemScheme, _fileSystemRoots, _fileSystem.path.separator == r'\');
     }
-    mainUri ??= toMultiRootPath(mainFileUri, _fileSystemScheme, _fileSystemRoots, _fileSystem.path.separator == r'\');
     if (outputFilePath != null && !_fileSystem.isFileSync(outputFilePath)) {
       _fileSystem.file(outputFilePath).createSync(recursive: true);
     }
@@ -359,7 +361,8 @@ class KernelCompiler {
       // See: https://github.com/flutter/flutter/issues/103994
       '--verbosity=error',
       ...?extraFrontEndOptions,
-      mainUri,
+      if (mainUri != null) mainUri
+      else '--native-assets-only',
     ];
 
     _logger.printTrace(command.join(' '));