Create end to end integration test for DevTools

.github/workflows/build.yaml

@@ -155,6 +155,29 @@ jobs:
           name: golden_image_failures.${{ matrix.bot }}
           path: packages/devtools_app/test/**/failures/*.png
 
+  integration-test:
+    name: integration-test ${{ matrix.bot }}
+    needs: flutter-prep
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        bot:
+          # Consider running integration tests in ddc mode, too.
+          - integration_dart2js
+    steps:
+      - name: git clone
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
+      - name: Load Cached Flutter SDK
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./flutter-sdk
+          key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} 
+      - name: tool/bots.sh
+        env:
+          BOT: ${{ matrix.bot }}
+        run: ./tool/bots.sh
 
 # TODO(https://github.com/flutter/devtools/issues/1715): add a windows compatible version of tool/bots.sh
 # and run it from this job.

packages/devtools_app/integration_test/test/app_test.dart

@@ -0,0 +1,74 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:devtools_app/devtools_app.dart';
+import 'package:devtools_app/src/app.dart';
+import 'package:devtools_app/src/framework/landing_screen.dart';
+import 'package:devtools_app/src/framework/release_notes/release_notes.dart';
+import 'package:devtools_app/src/shared/primitives/simple_items.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/integration_test.dart';
+
+import 'test_utils.dart';
+
+void main() {
+  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+
+  late TestApp testApp;
+
+  setUpAll(() {
+    testApp = TestApp.fromEnvironment();
+    expect(testApp.vmServiceUri, isNotNull);
+  });
+
+  testWidgets('connect to app and switch tabs', (tester) async {
+    await pumpDevTools(tester);
+    expect(find.byType(LandingScreenBody), findsOneWidget);
+    expect(find.text('No client connection'), findsOneWidget);
+
+    logStatus('verify that we can connect to an app');
+    await connectToTestApp(tester, testApp);
+    expect(find.byType(LandingScreenBody), findsNothing);
+    expect(find.text('No client connection'), findsNothing);
+
+    // If the release notes viewer is open, close it.
+    final releaseNotesView =
+        tester.widget<ReleaseNotes>(find.byType(ReleaseNotes));
+    if (releaseNotesView.releaseNotesController.releaseNotesVisible.value) {
+      final closeReleaseNotesButton = find.descendant(
+        of: find.byType(ReleaseNotes),
+        matching: find.byType(IconButton),
+      );
+      expect(closeReleaseNotesButton, findsOneWidget);
+      await tester.tap(closeReleaseNotesButton);
+    }
+
+    logStatus('verify that we can load each DevTools screen');
+    final availableScreenIds = <String>[];
+    for (final screen in devtoolsScreens!) {
+      if (shouldShowScreen(screen.screen)) {
+        availableScreenIds.add(screen.screen.screenId);
+      }
+    }
+    final tabs = tester.widgetList<Tab>(
+      find.descendant(
+        of: find.byType(AppBar),
+        matching: find.byType(Tab),
+      ),
+    );
+    expect(tabs.length, equals(availableScreenIds.length));
+
+    final screenTitles = (ScreenMetaData.values.toList()
+          ..removeWhere((data) => !availableScreenIds.contains(data.id)))
+        .map((data) => data.title);
+    for (final title in screenTitles) {
+      logStatus('switching to $title screen');
+      await tester.tap(find.widgetWithText(Tab, title));
+      // We use pump here instead of pumpAndSettle because pumpAndSettle will
+      // never complete if there is an animation (e.g. a progress indicator).
+      await tester.pump(safePumpDuration);
+    }
+  });
+}

packages/devtools_app/integration_test/test/test_utils.dart

@@ -0,0 +1,70 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:devtools_app/main.dart' as app;
+import 'package:devtools_app/src/app.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+const safePumpDuration = Duration(seconds: 3);
+const longPumpDuration = Duration(seconds: 6);
+
+Future<void> pumpDevTools(WidgetTester tester) async {
+  // TODO(kenz): how can we share code across integration_test/test and
+  // integration_test/test_infra? When trying to import, we get an error:
+  // Error when reading 'org-dartlang-app:/test_infra/shared.dart': File not found
+  const shouldEnableExperiments = bool.fromEnvironment('enable_experiments');
+  await app.runDevTools(
+    // ignore: avoid_redundant_argument_values
+    shouldEnableExperiments: shouldEnableExperiments,
+  );
+
+  // Await a delay to ensure the widget tree has loaded.
+  await tester.pumpAndSettle(longPumpDuration);
+  expect(find.byType(DevToolsApp), findsOneWidget);
+}
+
+Future<void> connectToTestApp(WidgetTester tester, TestApp testApp) async {
+  final textFieldFinder = find.byType(TextField);
+  // TODO(https://github.com/flutter/flutter/issues/89749): use
+  // `tester.enterText` once this issue is fixed.
+  (tester.firstWidget(textFieldFinder) as TextField).controller?.text =
+      testApp.vmServiceUri;
+  await tester.tap(
+    find.ancestor(
+      of: find.text('Connect'),
+      matching: find.byType(ElevatedButton),
+    ),
+  );
+  await tester.pumpAndSettle(safePumpDuration);
+}
+
+void logStatus(String log) {
+  print('TEST STATUS: $log');
+}
+
+class TestApp {
+  TestApp._({required this.vmServiceUri});
+
+  factory TestApp.parse(Map<String, Object> json) {
+    final serviceUri = json[serviceUriKey] as String?;
+    if (serviceUri == null) {
+      throw Exception('Cannot create a TestApp with a null service uri.');
+    }
+    return TestApp._(vmServiceUri: serviceUri);
+  }
+
+  factory TestApp.fromEnvironment() {
+    const testArgs = String.fromEnvironment('test_args');
+    final Map<String, Object> argsMap =
+        jsonDecode(testArgs).cast<String, Object>();
+    return TestApp.parse(argsMap);
+  }
+
+  static const serviceUriKey = 'service_uri';
+
+  final String vmServiceUri;
+}

packages/devtools_app/integration_test/test_infra/_run_test.dart

@@ -23,8 +23,12 @@ Future<void> runFlutterIntegrationTest(
   if (shouldCreateTestApp) {
     // Create the test app and start it.
     // TODO(kenz): support running Dart CLI test apps from here too.
-    testApp = TestFlutterApp(appPath: testAppPath);
-    await testApp.start();
+    try {
+      testApp = TestFlutterApp(appPath: testAppPath);
+      await testApp.start();
+    } catch (e) {
+      throw Exception('Error starting test app: $e');
+    }
     testAppUri = testApp.vmServiceUri.toString();
   } else {
     testAppUri = testRunnerArgs.testAppUri!;
@@ -33,30 +37,38 @@ Future<void> runFlutterIntegrationTest(
   // TODO(kenz): do we need to start chromedriver in headless mode?
   // Start chrome driver before running the flutter integration test.
   final chromedriver = ChromeDriver();
-  await chromedriver.start();
+  try {
+    await chromedriver.start();
+  } catch (e) {
+    throw Exception('Error starting chromedriver: $e');
+  }
 
   // Run the flutter integration test.
   final testRunner = TestRunner();
-  await testRunner.run(
-    testRunnerArgs.testTarget,
-    enableExperiments: testRunnerArgs.enableExperiments,
-    headless: testRunnerArgs.headless,
-    testAppArguments: {
-      'service_uri': testAppUri,
-    },
-  );
-
-  if (shouldCreateTestApp) {
-    _debugLog('killing the test app');
-    await testApp?.killGracefully();
+  try {
+    await testRunner.run(
+      testRunnerArgs.testTarget,
+      enableExperiments: testRunnerArgs.enableExperiments,
+      headless: testRunnerArgs.headless,
+      testAppArguments: {
+        'service_uri': testAppUri,
+      },
+    );
+  } catch (_) {
+    rethrow;
+  } finally {
+    if (shouldCreateTestApp) {
+      _debugLog('killing the test app');
+      await testApp?.stop();
+    }
+
+    _debugLog('cancelling stream subscriptions');
+    await testRunner.cancelAllStreamSubscriptions();
+    await chromedriver.cancelAllStreamSubscriptions();
+
+    _debugLog('killing the chromedriver process');
+    chromedriver.kill();
   }
-
-  _debugLog('cancelling stream subscriptions');
-  await testRunner.cancelAllStreamSubscriptions();
-  await chromedriver.cancelAllStreamSubscriptions();
-
-  _debugLog('killing the chromedriver process');
-  chromedriver.kill();
 }
 
 class ChromeDriver with IOMixin {
@@ -103,14 +115,62 @@ class TestRunner with IOMixin {
         if (enableExperiments) '--dart-define=enable_experiments=true',
       ],
     );
-    listenToProcessOutput(process);
+    listenToProcessOutput(
+      process,
+      onStdout: (line) {
+        if (line.startsWith(_TestResult.testResultPrefix)) {
+          final testResultJson = line.substring(line.indexOf('{'));
+          final testResultMap =
+              jsonDecode(testResultJson) as Map<String, Object?>;
+          final result = _TestResult.parse(testResultMap);
+          if (!result.result) {
+            throw Exception(result.toString());
+          }
+        }
+        print('stdout = $line');
+      },
+    );
 
     await process.exitCode;
     process.kill();
     _debugLog('flutter drive process has exited');
   }
 }
 
+class _TestResult {
+  _TestResult._(this.result, this.methodName, this.details);
+
+  factory _TestResult.parse(Map<String, Object?> json) {
+    final result = json[resultKey] == 'true';
+    final failureDetails =
+        (json[failureDetailsKey] as List<Object?>).cast<String>().firstOrNull ??
+            '{}';
+    final failureDetailsMap =
+        jsonDecode(failureDetails) as Map<String, Object?>;
+    final methodName = failureDetailsMap[methodNameKey] as String?;
+    final details = failureDetailsMap[detailsKey] as String?;
+    return _TestResult._(result, methodName, details);
+  }
+
+  static const testResultPrefix = 'result {"result":';
+  static const resultKey = 'result';
+  static const failureDetailsKey = 'failureDetails';
+  static const methodNameKey = 'methodName';
+  static const detailsKey = 'details';
+
+  final bool result;
+  final String? methodName;
+  final String? details;
+
+  @override
+  String toString() {
+    if (result) {
+      return 'Test passed';
+    }
+    return 'Test \'$methodName\' failed: $details.';
+  }
+}
+
 void _debugLog(String log) {
   if (_debugTestScript) {
     print(log);

packages/devtools_app/integration_test/test_infra/io_utils.dart

@@ -22,12 +22,18 @@ mixin IOMixin {
   void listenToProcessOutput(
     Process process, {
     void Function(String) printCallback = _defaultPrintCallback,
+    void Function(String)? onStdout,
+    void Function(String)? onStderr,
   }) {
     streamSubscriptions.addAll([
-      transformToLines(process.stdout)
-          .listen((String line) => stdoutController.add(line)),
-      transformToLines(process.stderr)
-          .listen((String line) => stderrController.add(line)),
+      transformToLines(process.stdout).listen((String line) {
+        onStdout?.call(line);
+        stdoutController.add(line);
+      }),
+      transformToLines(process.stderr).listen((String line) {
+        onStderr?.call(line);
+        stderrController.add(line);
+      }),
 
       // This is just debug printing to aid running/debugging tests locally.
       stdoutController.stream.listen(printCallback),