Implement hot reload using the DDC library bundle format (flutter#162498)

dart-lang/webdev#2516

- Updates restart/reload code to accept a resetCompiler boolean to
disambiguate between whether this is a full restart and whether to reset
the resident compiler.
- Adds code to call reloadSources in DWDS and handle the response
(including any errors).
- Adds code to invoke reassemble.
- Adds code to emit a script that DWDS can later consume that contains
the changed sources and their associated libraries. This is used to hot
reload. The bootstrapper puts this in the global window. DWDS should be
updated to accept it in the provider itself. See
dart-lang/webdev#2584.
- Adds code to parse module metadata from the frontend server. This is
identical to the implementation in DWDS % addressing type-related lints.
- Adds tests that run the existing hot reload tests but with web. Some
modifications are mode, including waiting for Flutter runs to finish
executing, and skipping a test that's not possible on the web.

Needs DWDS 24.3.4 to be published first and used before we can land.

## Pre-launch Checklist

- [x] I read the [Contributor Guide] and followed the process outlined
there for submitting PRs.
- [x] I read the [Tree Hygiene] wiki page, which explains my
responsibilities.
- [x] I read and followed the [Flutter Style Guide], including [Features
we expect every widget to implement].
- [x] I signed the [CLA].
- [x] I listed at least one issue that this PR fixes in the description
above.
- [x] I updated/added relevant documentation (doc comments with `///`).
- [x] I added new tests to check the change I am making, or this PR is
[test-exempt].
- [x] I followed the [breaking change policy] and added [Data Driven
Fixes] where supported.
- [ ] All existing and new tests are passing.

Author: srujzs

packages/flutter_tools/lib/src/devfs.dart

@@ -551,6 +551,10 @@ class DevFS {
   /// Updates files on the device.
   ///
   /// Returns the number of bytes synced.
+  ///
+  /// If [fullRestart] is true, assumes this is a hot restart instead of a hot
+  /// reload. If [resetCompiler] is true, sends a `reset` instruction to the
+  /// frontend server.
   Future<UpdateFSReport> update({
     required Uri mainUri,
     required ResidentCompiler generator,
@@ -566,6 +570,7 @@ class DevFS {
     AssetBundle? bundle,
     bool bundleFirstUpload = false,
     bool fullRestart = false,
+    bool resetCompiler = false,
     File? dartPluginRegistrant,
   }) async {
     final DateTime candidateCompileTime = DateTime.now();
@@ -577,7 +582,7 @@ class DevFS {
     final List<Future<void>> pendingAssetBuilds = <Future<void>>[];
     bool assetBuildFailed = false;
     int syncedBytes = 0;
-    if (fullRestart) {
+    if (resetCompiler) {
       generator.reset();
     }
     // On a full restart, or on an initial compile for the attach based workflow,

packages/flutter_tools/lib/src/isolated/devfs_web.dart

@@ -40,6 +40,7 @@ import '../web/bootstrap.dart';
 import '../web/chrome.dart';
 import '../web/compile.dart';
 import '../web/memory_fs.dart';
+import '../web/module_metadata.dart';
 import '../web_template.dart';
 
 typedef DwdsLauncher =
@@ -158,6 +159,16 @@ class WebAssetServer implements AssetReader {
   /// If [writeRestartScripts] is true, writes a list of sources mapped to their
   /// ids to the file system that can then be consumed by the hot restart
   /// callback.
+  ///
+  /// For example:
+  /// ```json
+  /// [
+  ///   {
+  ///     "src": "<file_name>",
+  ///     "id": "<id>",
+  ///   },
+  /// ]
+  /// ```
   void performRestart(List<String> modules, {required bool writeRestartScripts}) {
     for (final String module in modules) {
       // We skip computing the digest by using the hashCode of the underlying buffer.
@@ -174,11 +185,44 @@ class WebAssetServer implements AssetReader {
       for (final String src in modules) {
         srcIdsList.add(<String, String>{'src': '$src?gen=$_hotRestartGeneration', 'id': src});
       }
-      writeFile('main.dart.js.restartScripts', json.encode(srcIdsList));
+      writeFile('restart_scripts.json', json.encode(srcIdsList));
     }
     _hotRestartGeneration++;
   }
 
+  /// Given a list of [modules] that need to be reloaded, writes a file that
+  /// contains a list of objects each with two fields:
+  ///
+  /// `src`: A string that corresponds to the file path containing a DDC library
+  /// bundle.
+  /// `libraries`: An array of strings containing the libraries that were
+  /// compiled in `src`.
+  ///
+  /// For example:
+  /// ```json
+  /// [
+  ///   {
+  ///     "src": "<file_name>",
+  ///     "libraries": ["<lib1>", "<lib2>"],
+  ///   },
+  /// ]
+  /// ```
+  ///
+  /// The path of the output file should stay consistent across the lifetime of
+  /// the app.
+  void performReload(List<String> modules) {
+    final List<Map<String, Object>> moduleToLibrary = <Map<String, Object>>[];
+    for (final String module in modules) {
+      final ModuleMetadata metadata = ModuleMetadata.fromJson(
+        json.decode(utf8.decode(_webMemoryFS.metadataFiles['$module.metadata']!.toList()))
+            as Map<String, dynamic>,
+      );
+      final List<String> libraries = metadata.libraries.keys.toList();
+      moduleToLibrary.add(<String, Object>{'src': module, 'libraries': libraries});
+    }
+    writeFile('reload_scripts.json', json.encode(moduleToLibrary));
+  }
+
   @visibleForTesting
   List<String> write(File codeFile, File manifestFile, File sourcemapFile, File metadataFile) {
     return _webMemoryFS.write(codeFile, manifestFile, sourcemapFile, metadataFile);
@@ -1001,6 +1045,7 @@ class WebDevFS implements DevFS {
     AssetBundle? bundle,
     bool bundleFirstUpload = false,
     bool fullRestart = false,
+    bool resetCompiler = false,
     String? projectRootPath,
     File? dartPluginRegistrant,
   }) async {
@@ -1077,7 +1122,7 @@ class WebDevFS implements DevFS {
     await _validateTemplateFile('index.html');
     await _validateTemplateFile('flutter_bootstrap.js');
     final DateTime candidateCompileTime = DateTime.now();
-    if (fullRestart) {
+    if (resetCompiler) {
       generator.reset();
     }
 
@@ -1122,8 +1167,11 @@ class WebDevFS implements DevFS {
     } on FileSystemException catch (err) {
       throwToolExit('Failed to load recompiled sources:\n$err');
     }
-
-    webAssetServer.performRestart(modules, writeRestartScripts: ddcModuleSystem);
+    if (fullRestart) {
+      webAssetServer.performRestart(modules, writeRestartScripts: ddcModuleSystem);
+    } else {
+      webAssetServer.performReload(modules);
+    }
     return UpdateFSReport(
       success: true,
       syncedBytes: codeFile.lengthSync(),

packages/flutter_tools/lib/src/isolated/resident_web_runner.dart

@@ -322,7 +322,7 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
         }
         if (debuggingOptions.buildInfo.isDebug && !debuggingOptions.webUseWasm) {
           await runSourceGenerators();
-          final UpdateFSReport report = await _updateDevFS(fullRestart: true);
+          final UpdateFSReport report = await _updateDevFS(fullRestart: true, resetCompiler: true);
           if (!report.success) {
             _logger.printError('Failed to compile application.');
             appFailedToStart();
@@ -406,15 +406,28 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
     bool benchmarkMode = false,
   }) async {
     final DateTime start = _systemClock.now();
-    final Status status = _logger.startProgress(
-      'Performing hot restart...',
-      progressId: 'hot.restart',
-    );
+    final Status status;
+    if (debuggingOptions.buildInfo.ddcModuleFormat != DdcModuleFormat.ddc ||
+        debuggingOptions.buildInfo.canaryFeatures == false) {
+      // Triggering hot reload performed hot restart for the old module formats
+      // historically. Keep that behavior and only perform hot reload when the
+      // new module format is used.
+      fullRestart = true;
+    }
+    if (fullRestart) {
+      status = _logger.startProgress('Performing hot restart...', progressId: 'hot.restart');
+    } else {
+      status = _logger.startProgress('Performing hot reload...', progressId: 'hot.reload');
+    }
 
     if (debuggingOptions.buildInfo.isDebug && !debuggingOptions.webUseWasm) {
       await runSourceGenerators();
-      // Full restart is always false for web, since the extra recompile is wasteful.
-      final UpdateFSReport report = await _updateDevFS();
+      // Don't reset the resident compiler for web, since the extra recompile is
+      // wasteful.
+      final UpdateFSReport report = await _updateDevFS(
+        fullRestart: fullRestart,
+        resetCompiler: false,
+      );
       if (report.success) {
         device!.generator!.accept();
       } else {
@@ -448,10 +461,32 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
       if (!deviceIsDebuggable) {
         _logger.printStatus('Recompile complete. Page requires refresh.');
       } else if (isRunningDebug) {
-        // If the hot-restart service extension method is registered, then use
-        // it. Otherwise, default to calling "hotRestart" without a namespace.
-        final String hotRestartMethod = _registeredMethodsForService['hotRestart'] ?? 'hotRestart';
-        await _vmService.service.callMethod(hotRestartMethod);
+        if (fullRestart) {
+          // If the hot-restart service extension method is registered, then use
+          // it. Otherwise, default to calling "hotRestart" without a namespace.
+          final String hotRestartMethod =
+              _registeredMethodsForService['hotRestart'] ?? 'hotRestart';
+          await _vmService.service.callMethod(hotRestartMethod);
+        } else {
+          // Isolates don't work on web. For lack of a better value, pass an
+          // empty string for the isolate id.
+          final vmservice.ReloadReport report = await _vmService.service.reloadSources('');
+          final ReloadReportContents contents = ReloadReportContents.fromReloadReport(report);
+          final bool success = contents.success ?? false;
+          if (!success) {
+            // Rejections happen at compile-time for the web, so in theory,
+            // nothing should go wrong here. However, if DWDS or the DDC runtime
+            // has some internal error, we should still surface it to make
+            // debugging easier.
+            String reloadFailedMessage = 'Hot reload failed:';
+            globals.printError(reloadFailedMessage);
+            for (final ReasonForCancelling reason in contents.notices) {
+              reloadFailedMessage += reason.toString();
+              globals.printError(reason.toString());
+            }
+            return OperationResult(1, reloadFailedMessage);
+          }
+        }
       } else {
         // On non-debug builds, a hard refresh is required to ensure the
         // up to date sources are loaded.
@@ -467,40 +502,76 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
 
     final Duration elapsed = _systemClock.now().difference(start);
     final String elapsedMS = getElapsedAsMilliseconds(elapsed);
-    _logger.printStatus('Restarted application in $elapsedMS.');
+    _logger.printStatus('${fullRestart ? 'Restarted' : 'Reloaded'} application in $elapsedMS.');
 
-    unawaited(residentDevtoolsHandler!.hotRestart(flutterDevices));
+    if (fullRestart) {
+      unawaited(residentDevtoolsHandler!.hotRestart(flutterDevices));
+    }
 
     // Don't track restart times for dart2js builds or web-server devices.
     if (debuggingOptions.buildInfo.isDebug && deviceIsDebuggable) {
-      _analytics.send(
-        Event.timing(
-          workflow: 'hot',
-          variableName: 'web-incremental-restart',
-          elapsedMilliseconds: elapsed.inMilliseconds,
-        ),
-      );
+      // TODO(srujzs): There are a number of fields that the VM tracks in the
+      // analytics that we do not for both hot restart and reload. We should
+      // unify that.
+      final String targetPlatform = getNameForTargetPlatform(TargetPlatform.web_javascript);
       final String sdkName = await device!.device!.sdkNameAndVersion;
-      HotEvent(
-        'restart',
-        targetPlatform: getNameForTargetPlatform(TargetPlatform.web_javascript),
-        sdkName: sdkName,
-        emulator: false,
-        fullRestart: true,
-        reason: reason,
-        overallTimeInMs: elapsed.inMilliseconds,
-      ).send();
-      _analytics.send(
-        Event.hotRunnerInfo(
-          label: 'restart',
-          targetPlatform: getNameForTargetPlatform(TargetPlatform.web_javascript),
+      if (fullRestart) {
+        _analytics.send(
+          Event.timing(
+            workflow: 'hot',
+            variableName: 'web-incremental-restart',
+            elapsedMilliseconds: elapsed.inMilliseconds,
+          ),
+        );
+        HotEvent(
+          'restart',
+          targetPlatform: targetPlatform,
           sdkName: sdkName,
           emulator: false,
           fullRestart: true,
           reason: reason,
           overallTimeInMs: elapsed.inMilliseconds,
-        ),
-      );
+        ).send();
+        _analytics.send(
+          Event.hotRunnerInfo(
+            label: 'restart',
+            targetPlatform: targetPlatform,
+            sdkName: sdkName,
+            emulator: false,
+            fullRestart: true,
+            reason: reason,
+            overallTimeInMs: elapsed.inMilliseconds,
+          ),
+        );
+      } else {
+        _analytics.send(
+          Event.timing(
+            workflow: 'hot',
+            variableName: 'reload',
+            elapsedMilliseconds: elapsed.inMilliseconds,
+          ),
+        );
+        HotEvent(
+          'reload',
+          targetPlatform: targetPlatform,
+          sdkName: sdkName,
+          emulator: false,
+          fullRestart: false,
+          reason: reason,
+          overallTimeInMs: elapsed.inMilliseconds,
+        ).send();
+        _analytics.send(
+          Event.hotRunnerInfo(
+            label: 'reload',
+            targetPlatform: targetPlatform,
+            sdkName: sdkName,
+            emulator: false,
+            fullRestart: false,
+            reason: reason,
+            overallTimeInMs: elapsed.inMilliseconds,
+          ),
+        );
+      }
     }
     return OperationResult.ok;
   }
@@ -551,7 +622,10 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
     return result!.absolute.uri;
   }
 
-  Future<UpdateFSReport> _updateDevFS({bool fullRestart = false}) async {
+  Future<UpdateFSReport> _updateDevFS({
+    required bool fullRestart,
+    required bool resetCompiler,
+  }) async {
     final bool isFirstUpload = !assetBundle.wasBuiltOnce();
     final bool rebuildBundle = assetBundle.needsBuild();
     if (rebuildBundle) {
@@ -584,8 +658,9 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
       bundleFirstUpload: isFirstUpload,
       generator: device!.generator!,
       fullRestart: fullRestart,
+      resetCompiler: resetCompiler,
       dillOutputPath: dillOutputPath,
-      pathToReload: getReloadPath(fullRestart: fullRestart, swap: false),
+      pathToReload: getReloadPath(resetCompiler: resetCompiler, swap: false),
       invalidatedFiles: invalidationResult.uris!,
       packageConfig: invalidationResult.packageConfig!,
       trackWidgetCreation: debuggingOptions.buildInfo.trackWidgetCreation,

packages/flutter_tools/lib/src/resident_runner.dart

@@ -570,6 +570,7 @@ class FlutterDevice {
         bundleFirstUpload: bundleFirstUpload,
         generator: generator!,
         fullRestart: fullRestart,
+        resetCompiler: fullRestart,
         dillOutputPath: dillOutputPath,
         trackWidgetCreation: buildInfo.trackWidgetCreation,
         pathToReload: pathToReload,
@@ -1112,8 +1113,8 @@ abstract class ResidentRunner extends ResidentHandlers {
 
   String get dillOutputPath =>
       _dillOutputPath ?? globals.fs.path.join(artifactDirectory.path, 'app.dill');
-  String getReloadPath({bool fullRestart = false, required bool swap}) {
-    if (!fullRestart) {
+  String getReloadPath({bool resetCompiler = false, required bool swap}) {
+    if (!resetCompiler) {
       return 'main.dart.incremental.dill';
     }
     return 'main.dart${swap ? '.swap' : ''}.dill';

packages/flutter_tools/lib/src/run_hot.dart

@@ -536,7 +536,7 @@ class HotRunner extends ResidentRunner {
           bundleFirstUpload: isFirstUpload,
           bundleDirty: !isFirstUpload && rebuildBundle,
           fullRestart: fullRestart,
-          pathToReload: getReloadPath(fullRestart: fullRestart, swap: _swap),
+          pathToReload: getReloadPath(resetCompiler: fullRestart, swap: _swap),
           invalidatedFiles: invalidationResult.uris!,
           packageConfig: invalidationResult.packageConfig!,
           dillOutputPath: dillOutputPath,

packages/flutter_tools/lib/src/web/bootstrap.dart

@@ -252,7 +252,15 @@ $_simpleLoaderScript
     // We should have written a file containing all the scripts that need to be
     // reloaded into the page. This is then read when a hot restart is triggered
     // in DDC via the `\$dartReloadModifiedModules` callback.
-    let restartScripts = currentUri + '.restartScripts';
+    let restartScripts = _currentDirectory + 'restart_scripts.json';
+    // Flutter tools should write a file containing the scripts and libraries
+    // that need to be hot reloaded. This is read in DWDS when a hot reload is
+    // triggered.
+    // TODO(srujzs): Ideally, this should be passed to the
+    // `FrontendServerDdcLibraryBundleStrategyProvider` instead. See
+    // https://github.com/dart-lang/webdev/issues/2584 for more details.
+    let reloadScripts = _currentDirectory + 'reload_scripts.json';
+    window.\$reloadScriptsPath = reloadScripts;
 
     if (!window.\$dartReloadModifiedModules) {
       window.\$dartReloadModifiedModules = (function(appName, callback) {