Investigate benchmarks run failure on CI

packages/devtools_app/benchmark/devtools_benchmarks_test.dart

@@ -0,0 +1,86 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Note: this test was modeled after the example test from Flutter Gallery:
+// https://github.com/flutter/gallery/blob/master/test_benchmarks/benchmarks_test.dart
+
+import 'dart:convert' show JsonEncoder;
+import 'dart:io';
+
+import 'package:test/test.dart';
+import 'package:web_benchmarks/server.dart';
+
+import 'test_infra/common.dart';
+import 'test_infra/project_root_directory.dart';
+
+final metricList = <String>[
+  'preroll_frame',
+  'apply_frame',
+  'drawFrameDuration',
+];
+
+final valueList = <String>[
+  'average',
+  'outlierAverage',
+  'outlierRatio',
+  'noise',
+];
+
+/// Tests that the DevTools web benchmarks are run and reported correctly.
+void main() {
+  test(
+    'Can run a web benchmark',
+    () async {
+      stdout.writeln('Starting web benchmark tests ...');
+
+      final taskResult = await serveWebBenchmark(
+        benchmarkAppDirectory: projectRootDirectory(),
+        entryPoint: 'benchmark/test_infra/client.dart',
+        useCanvasKit: true,
+        treeShakeIcons: false,
+        initialPage: benchmarkInitialPage,
+      );
+
+      stdout.writeln('Web benchmark tests finished.');
+
+      expect(
+        taskResult.scores.keys,
+        hasLength(DevToolsBenchmark.values.length),
+      );
+
+      for (final benchmarkName in DevToolsBenchmark.values.map((e) => e.id)) {
+        expect(
+          taskResult.scores[benchmarkName],
+          hasLength(metricList.length * valueList.length + 1),
+        );
+
+        for (final metricName in metricList) {
+          for (final valueName in valueList) {
+            expect(
+              taskResult.scores[benchmarkName]?.where(
+                (score) => score.metric == '$metricName.$valueName',
+              ),
+              hasLength(1),
+            );
+          }
+        }
+
+        expect(
+          taskResult.scores[benchmarkName]?.where(
+            (score) => score.metric == 'totalUiFrame.average',
+          ),
+          hasLength(1),
+        );
+      }
+
+      expect(
+        const JsonEncoder.withIndent('  ').convert(taskResult.toJson()),
+        isA<String>(),
+      );
+    },
+    timeout: Timeout.none,
+  );
+
+  // TODO(kenz): add tests that verify performance meets some expected threshold
+}

packages/devtools_app/benchmark/test_infra/client.dart

@@ -0,0 +1,25 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:web_benchmarks/client.dart';
+
+import 'common.dart';
+import 'devtools_recorder.dart';
+
+typedef RecorderFactory = Recorder Function();
+
+final Map<String, RecorderFactory> benchmarks = <String, RecorderFactory>{
+  DevToolsBenchmark.navigateThroughOfflineScreens.id: () => DevToolsRecorder(
+        benchmark: DevToolsBenchmark.navigateThroughOfflineScreens,
+      ),
+};
+
+/// Runs the client of the DevTools web benchmarks.
+///
+/// When the DevTools web benchmarks are run, the server builds an app with this
+/// file as the entry point (see `run_benchmarks.dart`). The app automates
+/// the DevTools web app, records some performance data, and reports them.
+Future<void> main() async {
+  await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage);
+}

packages/devtools_app/benchmark/test_infra/common.dart

@@ -0,0 +1,19 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// The initial page to load upon opening the DevTools benchmark app or
+/// reloading it in Chrome.
+//
+// We use an empty initial page so that the benchmark server does not attempt
+// to load the default page 'index.html', which will show up as "page not
+// found" in DevTools.
+const String benchmarkInitialPage = '';
+
+const String devtoolsBenchmarkPrefix = 'devtools';
+
+enum DevToolsBenchmark {
+  navigateThroughOfflineScreens;
+
+  String get id => '${devtoolsBenchmarkPrefix}_$name';
+}

packages/devtools_app/benchmark/test_infra/devtools_automator.dart

@@ -0,0 +1,110 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:devtools_app/devtools_app.dart';
+import 'package:devtools_test/helpers.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+import 'common.dart';
+
+/// A class that automates the DevTools web app.
+class DevToolsAutomater {
+  DevToolsAutomater({
+    required this.benchmark,
+    required this.stopWarmingUpCallback,
+  });
+
+  /// The current benchmark.
+  final DevToolsBenchmark benchmark;
+
+  /// A function to call when warm-up is finished.
+  ///
+  /// This function is intended to ask `Recorder` to mark the warm-up phase
+  /// as over.
+  final void Function() stopWarmingUpCallback;
+
+  /// Whether the automation has ended.
+  bool finished = false;
+
+  /// A widget controller for automation.
+  late LiveWidgetController controller;
+
+  /// The [DevToolsApp] widget with automation.
+  Widget createWidget() {
+    // There is no `catchError` here, because all errors are caught by
+    // the zone set up in `lib/web_benchmarks.dart` in `flutter/flutter`.
+    Future<void>.delayed(safePumpDuration, automateDevToolsGestures);
+    return DevToolsApp(
+      defaultScreens(),
+      AnalyticsController(enabled: false, firstRun: false),
+    );
+  }
+
+  Future<void> automateDevToolsGestures() async {
+    await warmUp();
+
+    switch (benchmark) {
+      case DevToolsBenchmark.navigateThroughOfflineScreens:
+        await _handleNavigateThroughOfflineScreens();
+    }
+
+    // At the end of the test, mark as finished.
+    finished = true;
+  }
+
+  /// Warm up the animation.
+  Future<void> warmUp() async {
+    _logStatus('Warming up.');
+
+    // Let animation stop.
+    await animationStops();
+
+    // Set controller.
+    controller = LiveWidgetController(WidgetsBinding.instance);
+
+    await controller.pumpAndSettle();
+
+    // TODO(kenz): investigate if we need to do something like the Flutter
+    // Gallery benchmark tests to warn up the Flutter engine.
+
+    // When warm-up finishes, inform the recorder.
+    stopWarmingUpCallback();
+
+    _logStatus('Warm-up finished.');
+  }
+
+  Future<void> _handleNavigateThroughOfflineScreens() async {
+    _logStatus('Navigate through offline DevTools tabs');
+    await navigateThroughDevToolsScreens(
+      controller,
+      runWithExpectations: false,
+    );
+    _logStatus('==== End navigate through offline DevTools tabs ====');
+  }
+}
+
+void _logStatus(String log) {
+  // ignore: avoid_print, intentional test logging.
+  print('==== $log ====');
+}
+
+const Duration _animationCheckingInterval = Duration(milliseconds: 50);
+
+Future<void> animationStops() async {
+  if (!WidgetsBinding.instance.hasScheduledFrame) return;
+
+  final Completer stopped = Completer<void>();
+
+  Timer.periodic(_animationCheckingInterval, (timer) {
+    if (!WidgetsBinding.instance.hasScheduledFrame) {
+      stopped.complete();
+      timer.cancel();
+    }
+  });
+
+  await stopped.future;
+}

packages/devtools_app/benchmark/test_infra/devtools_recorder.dart

@@ -0,0 +1,45 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:devtools_app/initialization.dart';
+import 'package:flutter/material.dart';
+import 'package:web_benchmarks/client.dart';
+
+import 'common.dart';
+import 'devtools_automator.dart';
+
+/// A recorder that measures frame building durations for the DevTools.
+class DevToolsRecorder extends WidgetRecorder {
+  DevToolsRecorder({required this.benchmark})
+      : super(name: benchmark.id, useCustomWarmUp: true);
+
+  /// The name of the DevTools benchmark to be run.
+  ///
+  /// See `common.dart` for the list of the names of all benchmarks.
+  final DevToolsBenchmark benchmark;
+
+  DevToolsAutomater? _devToolsAutomator;
+  bool get _finished => _devToolsAutomator?.finished ?? false;
+
+  /// Whether we should continue recording.
+  @override
+  bool shouldContinue() => !_finished || profile.shouldContinue();
+
+  /// Creates the [DevToolsAutomater] widget.
+  @override
+  Widget createWidget() {
+    _devToolsAutomator = DevToolsAutomater(
+      benchmark: benchmark,
+      stopWarmingUpCallback: profile.stopWarmingUp,
+    );
+    return _devToolsAutomator!.createWidget();
+  }
+
+  @override
+  Future<Profile> run() async {
+    // ignore: invalid_use_of_visible_for_testing_member, valid use for benchmark tests.
+    await initializeDevTools();
+    return super.run();
+  }
+}

packages/devtools_app/benchmark/test_infra/project_root_directory.dart

@@ -0,0 +1,31 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Note: this code was copied from Flutter gallery
+// https://github.com/flutter/gallery/blob/main/test_benchmarks/benchmarks/project_root_directory.dart
+
+import 'dart:io';
+import 'package:path/path.dart' as path;
+
+bool _hasPubspec(Directory directory) {
+  return directory.listSync().any(
+        (entity) =>
+            FileSystemEntity.isFileSync(entity.path) &&
+            path.basename(entity.path) == 'pubspec.yaml',
+      );
+}
+
+Directory projectRootDirectory() {
+  var current = Directory.current.absolute;
+
+  while (!_hasPubspec(current)) {
+    if (current.path == current.parent.path) {
+      throw Exception('Reached file system root when seeking project root.');
+    }
+
+    current = current.parent;
+  }
+
+  return current;
+}

packages/devtools_app/pubspec.yaml

@@ -79,6 +79,7 @@ dev_dependencies:
   mockito: ^5.4.1
   stager: ^1.0.1
   test: ^1.21.1
+  web_benchmarks: ^0.1.0+10
   webkit_inspection_protocol: ">=0.5.0 <2.0.0"
 
 flutter:

tool/ci/benchmark_tests.sh

@@ -10,5 +10,5 @@ set -ex
 source ./tool/ci/setup.sh
 
 pushd $DEVTOOLS_DIR/packages/devtools_app
-flutter test test_benchmarks/
+flutter test test/benchmarks/
 popd