Add skeleton for offline integration tests and add CLI flags

packages/devtools_app/integration_test/run_tests.dart

@@ -17,38 +17,64 @@ import 'test_infra/_run_test.dart';
 // --test-app-uri=<some vm service uri> - this will connect DevTools to the app
 //    you specify instead of spinning up a test app inside
 //    [runFlutterIntegrationTest].
+// --offline - indicates that we do not need to start a test app to run this
+//    test. This will take precedence if both --offline and --test-app-uri are
+//    present.
 // --enable-experiments - this will run the DevTools integration tests with
 //    DevTools experiments enabled (see feature_flags.dart)
+// --update-goldens - this will update the current golden images with the
+//   results from this test run
 // --headless - this will run the integration test on the 'web-server' device
 //    instead of the 'chrome' device, meaning you will not be able to see the
 //    integration test run in Chrome when running locally.
 
+const _testDirectory = 'integration_test/test';
+const _testSuffix = '_test.dart';
+const _offlineIndicator = 'integration_test/test/offline';
+const _experimentalIndicator = '/experimental/';
+
 void main(List<String> args) async {
-  final targetProvided =
-      args.firstWhereOrNull((arg) => arg.startsWith(TestArgs.testTargetArg)) !=
-          null;
-  if (targetProvided) {
+  final modifiableArgs = List.of(args);
+
+  final testTarget = modifiableArgs
+      .firstWhereOrNull((arg) => arg.startsWith(TestArgs.testTargetArg));
+  if (testTarget != null) {
     // Run the single test at this path.
-    final testRunnerArgs = TestArgs(args);
+    _maybeAddOfflineArgument(modifiableArgs, testTarget);
+    _maybeAddExperimentsArgument(modifiableArgs, testTarget);
+    final testRunnerArgs = TestArgs(modifiableArgs);
+
+    // TODO(kenz): add support for specifying a directory as the target instead
+    // of a single file.
     await runFlutterIntegrationTest(testRunnerArgs);
   } else {
-    // Ran all tests since a target test was not provided.
-    const testSuffix = '_test.dart';
-
-    // TODO(kenz): if we end up having several subdirectories under
-    // `integration_test/test`, we could allow the directory to be modified with
-    // an argument (e.g. --directory=integration_test/test/performance).
-    final testDirectory = Directory('integration_test/test');
+    // Run all tests since a target test was not provided.
+    final testDirectory = Directory(_testDirectory);
     final testFiles = testDirectory
-        .listSync()
-        .where((testFile) => testFile.path.endsWith(testSuffix));
+        .listSync(recursive: true)
+        .where((testFile) => testFile.path.endsWith(_testSuffix));
+
     for (final testFile in testFiles) {
       final testTarget = testFile.path;
+      _maybeAddOfflineArgument(modifiableArgs, testTarget);
+      _maybeAddExperimentsArgument(modifiableArgs, testTarget);
       final testRunnerArgs = TestArgs([
-        ...args,
+        ...modifiableArgs,
         '${TestArgs.testTargetArg}$testTarget',
       ]);
       await runFlutterIntegrationTest(testRunnerArgs);
     }
   }
 }
+
+void _maybeAddOfflineArgument(List<String> args, String testTarget) {
+  if (testTarget.startsWith(_offlineIndicator)) {
+    args.add(TestArgs.offlineArg);
+  }
+}
+
+void _maybeAddExperimentsArgument(List<String> args, String testTarget) {
+  if (testTarget.contains(_experimentalIndicator)) {
+    args.add(TestArgs.enableExperimentsArg);
+  }
+}

packages/devtools_app/integration_test/test/live_connection/README.md

@@ -0,0 +1,7 @@
+# What is a "live connection" integration test?
+
+Tests in this directory will run DevTools and connect it to a live Dart or Flutter
+application. 
+
+See also `integration_test/test/offline`, which contains tests that run DevTools
+without connecting it to a live application.

packages/devtools_app/integration_test/test/offline/README.md

@@ -0,0 +1,10 @@
+# What is an "offline" integration test?
+
+Tests in this directory will run DevTools without connecting it to a live application.
+Integration tests in this directory will load offline data for testing. This is useful
+for testing features that will not have stable data from a live application. For example,
+the Performance screen timeline data will never be stable with a live applicaiton, so
+loading offline data allows for screenshot testing without flakiness.
+
+See also `integration_test/test/live_connection`, which contains tests that run DevTools
+and connect it to a live Dart or Flutter application. 

packages/devtools_app/integration_test/test_infra/_run_test.dart

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
@@ -19,19 +20,20 @@ Future<void> runFlutterIntegrationTest(
   TestFlutterApp? testApp;
   late String testAppUri;
 
-  final bool shouldCreateTestApp = testRunnerArgs.testAppUri == null;
-  if (shouldCreateTestApp) {
-    // Create the test app and start it.
-    // TODO(kenz): support running Dart CLI test apps from here too.
-    try {
-      testApp = TestFlutterApp(appPath: testAppPath);
-      await testApp.start();
-    } catch (e) {
-      throw Exception('Error starting test app: $e');
+  if (!testRunnerArgs.offline) {
+    if (testRunnerArgs.testAppUri == null) {
+      // Create the test app and start it.
+      // TODO(kenz): support running Dart CLI test apps from here too.
+      try {
+        testApp = TestFlutterApp(appPath: testAppPath);
+        await testApp.start();
+      } catch (e) {
+        throw Exception('Error starting test app: $e');
+      }
+      testAppUri = testApp.vmServiceUri.toString();
+    } else {
+      testAppUri = testRunnerArgs.testAppUri!;
     }
-    testAppUri = testApp.vmServiceUri.toString();
-  } else {
-    testAppUri = testRunnerArgs.testAppUri!;
   }
 
   // TODO(kenz): do we need to start chromedriver in headless mode?
@@ -45,21 +47,23 @@ Future<void> runFlutterIntegrationTest(
 
   // Run the flutter integration test.
   final testRunner = TestRunner();
+  Exception? exception;
   try {
     await testRunner.run(
       testRunnerArgs.testTarget,
       enableExperiments: testRunnerArgs.enableExperiments,
+      updateGoldens: testRunnerArgs.updateGoldens,
       headless: testRunnerArgs.headless,
       testAppArguments: {
-        'service_uri': testAppUri,
+        if (!testRunnerArgs.offline) 'service_uri': testAppUri,
       },
     );
-  } catch (_) {
-    rethrow;
+  } on Exception catch (e) {
+    exception = e;
   } finally {
-    if (shouldCreateTestApp) {
+    if (testApp != null) {
       _debugLog('killing the test app');
-      await testApp?.stop();
+      await testApp.stop();
     }
 
     _debugLog('cancelling stream subscriptions');
@@ -69,6 +73,10 @@ Future<void> runFlutterIntegrationTest(
     _debugLog('killing the chromedriver process');
     chromedriver.kill();
   }
+
+  if (exception != null) {
+    throw exception;
+  }
 }
 
 class ChromeDriver with IOMixin {
@@ -94,17 +102,23 @@ class ChromeDriver with IOMixin {
 }
 
 class TestRunner with IOMixin {
+  static const _beginExceptionMarker = '| EXCEPTION CAUGHT';
+  static const _endExceptionMarker = '===========================';
+
   Future<void> run(
     String testTarget, {
     bool headless = false,
     bool enableExperiments = false,
+    bool updateGoldens = false,
     Map<String, Object> testAppArguments = const <String, Object>{},
   }) async {
     _debugLog('starting the flutter drive process');
     final process = await Process.start(
       'flutter',
       [
         'drive',
+        // Note that debug outputs from the test will not show up in profile
+        // mode. See https://github.com/flutter/flutter/issues/69070.
         '--profile',
         '--driver=test_driver/integration_test.dart',
         '--target=$testTarget',
@@ -113,8 +127,13 @@ class TestRunner with IOMixin {
         if (testAppArguments.isNotEmpty)
           '--dart-define=test_args=${jsonEncode(testAppArguments)}',
         if (enableExperiments) '--dart-define=enable_experiments=true',
+        if (updateGoldens) '--dart-define=update_goldens=true',
       ],
     );
+
+    bool writeInProgress = false;
+    final exceptionBuffer = StringBuffer();
+
     listenToProcessOutput(
       process,
       onStdout: (line) {
@@ -124,16 +143,35 @@ class TestRunner with IOMixin {
               jsonDecode(testResultJson) as Map<String, Object?>;
           final result = _TestResult.parse(testResultMap);
           if (!result.result) {
-            throw Exception(result.toString());
+            exceptionBuffer
+              ..writeln('$result')
+              ..writeln();
+          }
+        }
+
+        if (line.contains(_beginExceptionMarker)) {
+          writeInProgress = true;
+        }
+        if (writeInProgress) {
+          exceptionBuffer.writeln(line);
+          // Marks the end of the exception caught by flutter.
+          if (line.contains(_endExceptionMarker) &&
+              !line.contains(_beginExceptionMarker)) {
+            writeInProgress = false;
+            exceptionBuffer.writeln();
           }
         }
-        print('stdout = $line');
       },
     );
 
     await process.exitCode;
+
     process.kill();
     _debugLog('flutter drive process has exited');
+
+    if (exceptionBuffer.isNotEmpty) {
+      throw Exception(exceptionBuffer.toString());
+    }
   }
 }
 
@@ -186,25 +224,54 @@ class TestArgs {
     final target = argWithTestTarget?.substring(testTargetArg.length);
     assert(
       target != null,
-      'Please specify a test target (e.g. --target=path/to/test.dart',
+      'Please specify a test target (e.g. ${testTargetArg}path/to/test.dart',
     );
     testTarget = target!;
 
     final argWithTestAppUri =
         args.firstWhereOrNull((arg) => arg.startsWith(testAppArg));
     testAppUri = argWithTestAppUri?.substring(testAppArg.length);
 
+    offline = args.contains(offlineArg);
     enableExperiments = args.contains(enableExperimentsArg);
+    updateGoldens = args.contains(updateGoldensArg);
     headless = args.contains(headlessArg);
   }
 
   static const testTargetArg = '--target=';
   static const testAppArg = '--test-app-uri=';
+  static const offlineArg = '--offline';
   static const enableExperimentsArg = '--enable-experiments';
+  static const updateGoldensArg = '--update-goldens';
   static const headlessArg = '--headless';
 
   late final String testTarget;
+
+  /// The Vm Service URI for the test app to connect devtools to.
+  ///
+  /// This value will only be used when [offline] has not been set to true.
   late final String? testAppUri;
+
+  /// Indicates that a test should not be run with a test app for connecting to
+  /// DevTools.
+  ///
+  /// When [offline] is true, the test will be responsible for loading offline
+  /// data to test DevTools against.
+  ///
+  /// `integration_test/run_tests.dart` will add this flag automatically for
+  /// test targets that lives under the integration_test/test/offline directory.
+  late final bool offline;
+
+  /// Whether DevTools experiments should be enabled for a test.
+  ///
+  /// `integration_test/run_tests.dart` will add this flag automatically for
+  /// test targets that lives under an `experimental/` directory.
   late final bool enableExperiments;
+
+  /// Whether golden images should be updated with the result of this test run.
+  late final bool updateGoldens;
+
+  /// Whether this integration test should be run on the 'web-server' device
+  /// instead of 'chrome'.
   late final bool headless;
 }