Add a basic performance benchmark test for DevTools.

.github/workflows/build.yaml

@@ -256,8 +256,8 @@ jobs:
           DEVTOOLS_PACKAGE: devtools_extensions
         run: ./tool/ci/bots.sh
 
-  benchmarks:
-    name: benchmarks
+  benchmark-performance:
+    name: benchmark-performance
     needs: flutter-prep
     runs-on: ubuntu-latest
     strategy:
@@ -271,8 +271,26 @@ jobs:
           path: |
             ./tool/flutter-sdk
           key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }}
-      - name: tool/ci/benchmark_tests.sh
-        run: ./tool/ci/benchmark_tests.sh
+      - name: tool/ci/benchmark_performance.sh
+        run: ./tool/ci/benchmark_performance.sh
+
+  benchmark-size:
+    name: benchmark-size
+    needs: flutter-prep
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - name: git clone
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+      - name: Load Cached Flutter SDK
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84
+        with:
+          path: |
+            ./tool/flutter-sdk
+          key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }}
+      - name: tool/ci/benchmark_size.sh
+        run: ./tool/ci/benchmark_size.sh
 
 
 # TODO(https://github.com/flutter/devtools/issues/1715): add a windows compatible version of tool/ci/bots.sh

packages/devtools_app/benchmark/README.md

@@ -0,0 +1,42 @@
+# DevTools benchmark tests
+
+There are two types of benchmarks that we currently support: size and performance.
+- devtools_benchmarks_test.dart (measures DevTools frame times)
+- web_bundle_size_test.dart (measures DevTools release build size)
+
+The benchmark tests are run automatically on the CI.
+See the "benchmark-performance" and "benchmark-size" jobs.
+
+## Running benchmark tests locally
+
+> [!NOTE] 
+> The performance and size benchmarks cannot be run concurrently
+> (e.g. by running `flutter test benchmark/`). See the [#caveats](#caveats)
+> section below.
+
+### Performance benchmarks
+
+To run the performance benchmark tests locally, run:
+```sh
+dart run run_benchmarks.dart
+```
+
+To run the test that verifies we can run benchmark tests, run:
+```sh
+flutter test benchmark/devtools_benchmarks_test.dart
+```
+
+### Size benchmarks
+
+To run the size benchmark test locally, run:
+```sh
+flutter test benchmark/web_bundle_size_test.dart
+```
+
+### Caveats
+
+The size benchmark must be ran by itself because it actually modifies the
+`devtools_app/build` folder to create and measure the release build web bundle size.
+If this test is ran while other tests are running, it can affect the measurements
+that the size benchmark test takes, and it can affect the DevTools build that
+the other running tests are using with.

packages/devtools_app/benchmark/devtools_benchmarks_test.dart

@@ -0,0 +1,86 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Note: this test was modeled after the example test from Flutter Gallery:
+// https://github.com/flutter/gallery/blob/master/test_benchmarks/benchmarks_test.dart
+
+import 'dart:convert' show JsonEncoder;
+import 'dart:io';
+
+import 'package:test/test.dart';
+import 'package:web_benchmarks/server.dart';
+
+import 'test_infra/common.dart';
+import 'test_infra/project_root_directory.dart';
+
+final metricList = <String>[
+  'preroll_frame',
+  'apply_frame',
+  'drawFrameDuration',
+];
+
+final valueList = <String>[
+  'average',
+  'outlierAverage',
+  'outlierRatio',
+  'noise',
+];
+
+/// Tests that the DevTools web benchmarks are run and reported correctly.
+void main() {
+  test(
+    'Can run a web benchmark',
+    () async {
+      stdout.writeln('Starting web benchmark tests ...');
+
+      final taskResult = await serveWebBenchmark(
+        benchmarkAppDirectory: projectRootDirectory(),
+        entryPoint: 'benchmark/test_infra/client.dart',
+        useCanvasKit: true,
+        treeShakeIcons: false,
+        initialPage: benchmarkInitialPage,
+      );
+
+      stdout.writeln('Web benchmark tests finished.');
+
+      expect(
+        taskResult.scores.keys,
+        hasLength(DevToolsBenchmark.values.length),
+      );
+
+      for (final benchmarkName in DevToolsBenchmark.values.map((e) => e.id)) {
+        expect(
+          taskResult.scores[benchmarkName],
+          hasLength(metricList.length * valueList.length + 1),
+        );
+
+        for (final metricName in metricList) {
+          for (final valueName in valueList) {
+            expect(
+              taskResult.scores[benchmarkName]?.where(
+                (score) => score.metric == '$metricName.$valueName',
+              ),
+              hasLength(1),
+            );
+          }
+        }
+
+        expect(
+          taskResult.scores[benchmarkName]?.where(
+            (score) => score.metric == 'totalUiFrame.average',
+          ),
+          hasLength(1),
+        );
+      }
+
+      expect(
+        const JsonEncoder.withIndent('  ').convert(taskResult.toJson()),
+        isA<String>(),
+      );
+    },
+    timeout: Timeout.none,
+  );
+
+  // TODO(kenz): add tests that verify performance meets some expected threshold
+}

packages/devtools_app/benchmark/run_benchmarks.dart

@@ -0,0 +1,30 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert' show JsonEncoder;
+import 'dart:io';
+
+import 'package:web_benchmarks/server.dart';
+
+import 'test_infra/common.dart';
+import 'test_infra/project_root_directory.dart';
+
+/// Runs the DevTools web benchmarks and reports the benchmark data.
+Future<void> main() async {
+  stdout.writeln('Starting web benchmark tests ...');
+
+  final taskResult = await serveWebBenchmark(
+    benchmarkAppDirectory: projectRootDirectory(),
+    entryPoint: 'benchmark/test_infra/client.dart',
+    useCanvasKit: true,
+    treeShakeIcons: false,
+    initialPage: benchmarkInitialPage,
+  );
+
+  stdout
+    ..writeln('Web benchmark tests finished.')
+    ..writeln('==== Results ====')
+    ..writeln(const JsonEncoder.withIndent('  ').convert(taskResult.toJson()))
+    ..writeln('==== End of results ====');
+}

packages/devtools_app/benchmark/test_infra/client.dart

@@ -0,0 +1,25 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:web_benchmarks/client.dart';
+
+import 'common.dart';
+import 'devtools_recorder.dart';
+
+typedef RecorderFactory = Recorder Function();
+
+final Map<String, RecorderFactory> benchmarks = <String, RecorderFactory>{
+  DevToolsBenchmark.navigateThroughOfflineScreens.id: () => DevToolsRecorder(
+        benchmark: DevToolsBenchmark.navigateThroughOfflineScreens,
+      ),
+};
+
+/// Runs the client of the DevTools web benchmarks.
+///
+/// When the DevTools web benchmarks are run, the server builds an app with this
+/// file as the entry point (see `run_benchmarks.dart`). The app automates
+/// the DevTools web app, records some performance data, and reports them.
+Future<void> main() async {
+  await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage);
+}

packages/devtools_app/benchmark/test_infra/common.dart

@@ -0,0 +1,19 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// The initial page to load upon opening the DevTools benchmark app or
+/// reloading it in Chrome.
+//
+// We use an empty initial page so that the benchmark server does not attempt
+// to load the default page 'index.html', which will show up as "page not
+// found" in DevTools.
+const String benchmarkInitialPage = '';
+
+const String devtoolsBenchmarkPrefix = 'devtools';
+
+enum DevToolsBenchmark {
+  navigateThroughOfflineScreens;
+
+  String get id => '${devtoolsBenchmarkPrefix}_$name';
+}

packages/devtools_app/benchmark/test_infra/devtools_automator.dart

@@ -0,0 +1,110 @@
+// Copyright 2023 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:devtools_app/devtools_app.dart';
+import 'package:devtools_test/helpers.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+import 'common.dart';
+
+/// A class that automates the DevTools web app.
+class DevToolsAutomater {
+  DevToolsAutomater({
+    required this.benchmark,
+    required this.stopWarmingUpCallback,
+  });
+
+  /// The current benchmark.
+  final DevToolsBenchmark benchmark;
+
+  /// A function to call when warm-up is finished.
+  ///
+  /// This function is intended to ask `Recorder` to mark the warm-up phase
+  /// as over.
+  final void Function() stopWarmingUpCallback;
+
+  /// Whether the automation has ended.
+  bool finished = false;
+
+  /// A widget controller for automation.
+  late LiveWidgetController controller;
+
+  /// The [DevToolsApp] widget with automation.
+  Widget createWidget() {
+    // There is no `catchError` here, because all errors are caught by
+    // the zone set up in `lib/web_benchmarks.dart` in `flutter/flutter`.
+    Future<void>.delayed(safePumpDuration, automateDevToolsGestures);
+    return DevToolsApp(
+      defaultScreens(),
+      AnalyticsController(enabled: false, firstRun: false),
+    );
+  }
+
+  Future<void> automateDevToolsGestures() async {
+    await warmUp();
+
+    switch (benchmark) {
+      case DevToolsBenchmark.navigateThroughOfflineScreens:
+        await _handleNavigateThroughOfflineScreens();
+    }
+
+    // At the end of the test, mark as finished.
+    finished = true;
+  }
+
+  /// Warm up the animation.
+  Future<void> warmUp() async {
+    _logStatus('Warming up.');
+
+    // Let animation stop.
+    await animationStops();
+
+    // Set controller.
+    controller = LiveWidgetController(WidgetsBinding.instance);
+
+    await controller.pumpAndSettle();
+
+    // TODO(kenz): investigate if we need to do something like the Flutter
+    // Gallery benchmark tests to warn up the Flutter engine.
+
+    // When warm-up finishes, inform the recorder.
+    stopWarmingUpCallback();
+
+    _logStatus('Warm-up finished.');
+  }
+
+  Future<void> _handleNavigateThroughOfflineScreens() async {
+    _logStatus('Navigate through offline DevTools tabs');
+    await navigateThroughDevToolsScreens(
+      controller,
+      runWithExpectations: false,
+    );
+    _logStatus('==== End navigate through offline DevTools tabs ====');
+  }
+}
+
+void _logStatus(String log) {
+  // ignore: avoid_print, intentional test logging.
+  print('==== $log ====');
+}
+
+const Duration _animationCheckingInterval = Duration(milliseconds: 50);
+
+Future<void> animationStops() async {
+  if (!WidgetsBinding.instance.hasScheduledFrame) return;
+
+  final Completer stopped = Completer<void>();
+
+  Timer.periodic(_animationCheckingInterval, (timer) {
+    if (!WidgetsBinding.instance.hasScheduledFrame) {
+      stopped.complete();
+      timer.cancel();
+    }
+  });
+
+  await stopped.future;
+}