Implement frame timing callbacks in Skwasm. (#50737)

Fixes flutter/flutter#140429

Some notes here:
* Refactored the frame timing systems so that we can deal with asynchronous rendering.
* Consolidated rendering of multiple pictures in skwasm into a single call, so that the rasterization can be properly measured.
* Pulled the frame timings tests into the `ui` test suite so that they run on all renderers (including skwasm).

Author: eyebrowsoffire

ci/licenses_golden/licenses_flutter

@@ -10324,6 +10324,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/font_fallback_data.dart + ../
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/font_fallbacks.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/fonts.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/frame_reference.dart + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/frame_timing_recorder.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/backdrop_filter.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/bitmap_canvas.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/canvas.dart + ../../../flutter/LICENSE
@@ -13161,6 +13162,7 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/font_fallback_data.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/font_fallbacks.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/fonts.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/frame_reference.dart
+FILE: ../../../flutter/lib/web_ui/lib/src/engine/frame_timing_recorder.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/backdrop_filter.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/bitmap_canvas.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/canvas.dart

lib/web_ui/lib/src/engine.dart

@@ -65,6 +65,7 @@ export 'engine/font_fallback_data.dart';
 export 'engine/font_fallbacks.dart';
 export 'engine/fonts.dart';
 export 'engine/frame_reference.dart';
+export 'engine/frame_timing_recorder.dart';
 export 'engine/html/backdrop_filter.dart';
 export 'engine/html/bitmap_canvas.dart';
 export 'engine/html/canvas.dart';

lib/web_ui/lib/src/engine/canvaskit/rasterizer.dart

@@ -131,6 +131,7 @@ abstract class DisplayCanvas {
 typedef RenderRequest = ({
   ui.Scene scene,
   Completer<void> completer,
+  FrameTimingRecorder? recorder,
 });
 
 /// A per-view queue of render requests. Only contains the current render

lib/web_ui/lib/src/engine/canvaskit/renderer.dart

@@ -417,16 +417,17 @@ class CanvasKitRenderer implements Renderer {
         "Unable to render to a view which hasn't been registered");
     final ViewRasterizer rasterizer = _rasterizers[view.viewId]!;
     final RenderQueue renderQueue = rasterizer.queue;
+    final FrameTimingRecorder? recorder = FrameTimingRecorder.frameTimingsEnabled ? FrameTimingRecorder() : null;
     if (renderQueue.current != null) {
       // If a scene is already queued up, drop it and queue this one up instead
       // so that the scene view always displays the most recently requested scene.
       renderQueue.next?.completer.complete();
       final Completer<void> completer = Completer<void>();
-      renderQueue.next = (scene: scene, completer: completer);
+      renderQueue.next = (scene: scene, completer: completer, recorder: recorder);
       return completer.future;
     }
     final Completer<void> completer = Completer<void>();
-    renderQueue.current = (scene: scene, completer: completer);
+    renderQueue.current = (scene: scene, completer: completer, recorder: recorder);
     unawaited(_kickRenderLoop(rasterizer));
     return completer.future;
   }
@@ -435,7 +436,7 @@ class CanvasKitRenderer implements Renderer {
     final RenderQueue renderQueue = rasterizer.queue;
     final RenderRequest current = renderQueue.current!;
     try {
-      await _renderScene(current.scene, rasterizer);
+      await _renderScene(current.scene, rasterizer, current.recorder);
       current.completer.complete();
     } catch (error, stackTrace) {
       current.completer.completeError(error, stackTrace);
@@ -449,19 +450,20 @@ class CanvasKitRenderer implements Renderer {
     }
   }
 
-  Future<void> _renderScene(ui.Scene scene, ViewRasterizer rasterizer) async {
+  Future<void> _renderScene(ui.Scene scene, ViewRasterizer rasterizer, FrameTimingRecorder? recorder) async {
     // "Build finish" and "raster start" happen back-to-back because we
     // render on the same thread, so there's no overhead from hopping to
     // another thread.
     //
     // CanvasKit works differently from the HTML renderer in that in HTML
     // we update the DOM in SceneBuilder.build, which is these function calls
     // here are CanvasKit-only.
-    frameTimingsOnBuildFinish();
-    frameTimingsOnRasterStart();
+    recorder?.recordBuildFinish();
+    recorder?.recordRasterStart();
 
     await rasterizer.draw((scene as LayerScene).layerTree);
-    frameTimingsOnRasterFinish();
+    recorder?.recordRasterFinish();
+    recorder?.submitTimings();
   }
 
   // Map from view id to the associated Rasterizer for that view.

lib/web_ui/lib/src/engine/dom.dart

@@ -1487,7 +1487,7 @@ class DomCanvasRenderingContextBitmapRenderer {}
 
 extension DomCanvasRenderingContextBitmapRendererExtension
     on DomCanvasRenderingContextBitmapRenderer {
-  external void transferFromImageBitmap(DomImageBitmap bitmap);
+  external void transferFromImageBitmap(DomImageBitmap? bitmap);
 }
 
 @JS('ImageData')

lib/web_ui/lib/src/engine/frame_timing_recorder.dart

@@ -0,0 +1,100 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:ui/src/engine.dart';
+import 'package:ui/ui.dart' as ui;
+
+class FrameTimingRecorder {
+  final int _vsyncStartMicros = _currentFrameVsyncStart;
+  final int _buildStartMicros = _currentFrameBuildStart;
+
+  int? _buildFinishMicros;
+  int? _rasterStartMicros;
+  int? _rasterFinishMicros;
+
+  /// Collects frame timings from frames.
+  ///
+  /// This list is periodically reported to the framework (see [_kFrameTimingsSubmitInterval]).
+  static List<ui.FrameTiming> _frameTimings = <ui.FrameTiming>[];
+
+  /// These two metrics are collected early in the process, before the respective
+  /// scene builders are created. These are instead treated as global state, which
+  /// are used to initialize any recorders that are created by the scene builders.
+  static int _currentFrameVsyncStart = 0;
+  static int _currentFrameBuildStart = 0;
+
+  static void recordCurrentFrameVsync() {
+    if (frameTimingsEnabled) {
+      _currentFrameVsyncStart = _nowMicros();
+    }
+  }
+
+  static void recordCurrentFrameBuildStart() {
+    if (frameTimingsEnabled) {
+      _currentFrameBuildStart = _nowMicros();
+    }
+  }
+
+  /// The last time (in microseconds) we submitted frame timings.
+  static int _frameTimingsLastSubmitTime = _nowMicros();
+  /// The amount of time in microseconds we wait between submitting
+  /// frame timings.
+  static const int _kFrameTimingsSubmitInterval = 100000; // 100 milliseconds
+
+  /// Whether we are collecting [ui.FrameTiming]s.
+  static bool get frameTimingsEnabled {
+    return EnginePlatformDispatcher.instance.onReportTimings != null;
+  }
+
+  /// Current timestamp in microseconds taken from the high-precision
+  /// monotonically increasing timer.
+  ///
+  /// See also:
+  ///
+  /// * https://developer.mozilla.org/en-US/docs/Web/API/Performance/now,
+  ///   particularly notes about Firefox rounding to 1ms for security reasons,
+  ///   which can be bypassed in tests by setting certain browser options.
+  static int _nowMicros() {
+    return (domWindow.performance.now() * 1000).toInt();
+  }
+
+  void recordBuildFinish([int? buildFinish]) {
+    assert(_buildFinishMicros == null, "can't record build finish more than once");
+    _buildFinishMicros = buildFinish ?? _nowMicros();
+  }
+
+  void recordRasterStart([int? rasterStart]) {
+    assert(_rasterStartMicros == null, "can't record raster start more than once");
+    _rasterStartMicros = rasterStart ?? _nowMicros();
+  }
+
+  void recordRasterFinish([int? rasterFinish]) {
+    assert(_rasterFinishMicros == null, "can't record raster finish more than once");
+    _rasterFinishMicros = rasterFinish ?? _nowMicros();
+  }
+
+  void submitTimings() {
+    assert(
+      _buildFinishMicros != null &&
+      _rasterStartMicros != null &&
+      _rasterFinishMicros != null,
+      'Attempted to submit an incomplete timings.'
+    );
+    final ui.FrameTiming timing = ui.FrameTiming(
+      vsyncStart: _vsyncStartMicros,
+      buildStart: _buildStartMicros,
+      buildFinish: _buildFinishMicros!,
+      rasterStart: _rasterStartMicros!,
+      rasterFinish: _rasterFinishMicros!,
+      rasterFinishWallTime: _rasterFinishMicros!,
+    );
+    _frameTimings.add(timing);
+    final int now = _nowMicros();
+    if (now - _frameTimingsLastSubmitTime > _kFrameTimingsSubmitInterval) {
+      _frameTimingsLastSubmitTime = now;
+      EnginePlatformDispatcher.instance.invokeOnReportTimings(_frameTimings);
+      _frameTimings = <ui.FrameTiming>[];
+    }
+  }
+}

lib/web_ui/lib/src/engine/html/renderer.dart

@@ -323,8 +323,11 @@ class HtmlRenderer implements Renderer {
   @override
   Future<void> renderScene(ui.Scene scene, ui.FlutterView view) async {
     final EngineFlutterView implicitView = EnginePlatformDispatcher.instance.implicitView!;
-    implicitView.dom.setScene((scene as SurfaceScene).webOnlyRootElement!);
-    frameTimingsOnRasterFinish();
+    scene as SurfaceScene;
+    implicitView.dom.setScene(scene.webOnlyRootElement!);
+    final FrameTimingRecorder? recorder = scene.timingRecorder;
+    recorder?.recordRasterFinish();
+    recorder?.submitTimings();
   }
 
   @override

lib/web_ui/lib/src/engine/html/scene.dart

@@ -2,22 +2,20 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'package:ui/src/engine/display.dart';
+import 'package:ui/src/engine.dart';
 import 'package:ui/ui.dart' as ui;
 
-import '../dom.dart';
-import '../vector_math.dart';
-import '../window.dart';
-import 'surface.dart';
-
 class SurfaceScene implements ui.Scene {
   /// This class is created by the engine, and should not be instantiated
   /// or extended directly.
   ///
   /// To create a Scene object, use a [SceneBuilder].
-  SurfaceScene(this.webOnlyRootElement);
+  SurfaceScene(this.webOnlyRootElement, {
+    required this.timingRecorder,
+  });
 
   final DomElement? webOnlyRootElement;
+  final FrameTimingRecorder? timingRecorder;
 
   /// Creates a raster image representation of the current state of the scene.
   /// This is a slow operation that is performed on a background thread.

lib/web_ui/lib/src/engine/html/scene_builder.dart

@@ -7,7 +7,7 @@ import 'dart:typed_data';
 import 'package:ui/ui.dart' as ui;
 import 'package:ui/ui_web/src/ui_web.dart' as ui_web;
 
-import '../../engine.dart' show kProfileApplyFrame, kProfilePrerollFrame;
+import '../../engine.dart' show FrameTimingRecorder, kProfileApplyFrame, kProfilePrerollFrame;
 import '../display.dart';
 import '../dom.dart';
 import '../picture.dart';
@@ -511,8 +511,9 @@ class SurfaceSceneBuilder implements ui.SceneBuilder {
     // In the HTML renderer we time the beginning of the rasterization phase
     // (counter-intuitively) in SceneBuilder.build because DOM updates happen
     // here. This is different from CanvasKit.
-    frameTimingsOnBuildFinish();
-    frameTimingsOnRasterStart();
+    final FrameTimingRecorder? recorder = FrameTimingRecorder.frameTimingsEnabled ? FrameTimingRecorder() : null;
+    recorder?.recordBuildFinish();
+    recorder?.recordRasterStart();
     timeAction<void>(kProfilePrerollFrame, () {
       while (_surfaceStack.length > 1) {
         // Auto-pop layers that were pushed without a corresponding pop.
@@ -528,7 +529,7 @@ class SurfaceSceneBuilder implements ui.SceneBuilder {
       }
       commitScene(_persistedScene);
       _lastFrameScene = _persistedScene;
-      return SurfaceScene(_persistedScene.rootElement);
+      return SurfaceScene(_persistedScene.rootElement, timingRecorder: recorder);
     });
   }
 

lib/web_ui/lib/src/engine/initialization.dart

@@ -158,7 +158,15 @@ Future<void> initializeEngineServices({
     if (!waitingForAnimation) {
       waitingForAnimation = true;
       domWindow.requestAnimationFrame((JSNumber highResTime) {
-        frameTimingsOnVsync();
+        FrameTimingRecorder.recordCurrentFrameVsync();
+
+        // In Flutter terminology "building a frame" consists of "beginning
+        // frame" and "drawing frame".
+        //
+        // We do not call `recordBuildFinish` from here because
+        // part of the rasterization process, particularly in the HTML
+        // renderer, takes place in the `SceneBuilder.build()`.
+        FrameTimingRecorder.recordCurrentFrameBuildStart();
 
         // Reset immediately, because `frameHandler` can schedule more frames.
         waitingForAnimation = false;
@@ -171,13 +179,6 @@ Future<void> initializeEngineServices({
         final int highResTimeMicroseconds =
             (1000 * highResTime.toDartDouble).toInt();
 
-        // In Flutter terminology "building a frame" consists of "beginning
-        // frame" and "drawing frame".
-        //
-        // We do not call `frameTimingsOnBuildFinish` from here because
-        // part of the rasterization process, particularly in the HTML
-        // renderer, takes place in the `SceneBuilder.build()`.
-        frameTimingsOnBuildStart();
         if (EnginePlatformDispatcher.instance.onBeginFrame != null) {
           EnginePlatformDispatcher.instance.invokeOnBeginFrame(
               Duration(microseconds: highResTimeMicroseconds));