Capture FAILURES!!! when running Android scenario_app tests. (#50255)

I expect `Linux linux_android_emulator_tests` and `Linux
linux_android_emulator_tests_api_33` to fail.

When they fail, I'll fix the assertion
(flutter/flutter#142746) before merging this
PR.

Author: matanlurey

.ci.yaml

@@ -60,6 +60,7 @@ targets:
       - DEPS
       - lib/ui/**
       - shell/platform/android/**
+      - testing/scenario_app/**
 
 # Task to run Linux linux_android_emulator_tests on AVDs running Android 33
 # instead of 34 for investigating https://github.com/flutter/flutter/issues/137947.
@@ -76,6 +77,7 @@ targets:
       - DEPS
       - lib/ui/**
       - shell/platform/android/**
+      - testing/scenario_app/**
 
   - name: Linux builder_cache
     enabled_branches:

testing/scenario_app/bin/android_integration_tests.dart

@@ -18,14 +18,49 @@ import 'utils/screenshot_transformer.dart';
 const int tcpPort = 3001;
 
 void main(List<String> args) async {
-  const ProcessManager pm = LocalProcessManager();
   final ArgParser parser = ArgParser()
-    ..addOption('adb', help: 'absolute path to the adb tool', mandatory: true)
-    ..addOption('out-dir', help: 'out directory', mandatory: true);
+    ..addOption(
+      'adb',
+      help: 'absolute path to the adb tool',
+      mandatory: true,
+    )
+    ..addOption(
+      'out-dir',
+      help: 'out directory',
+      mandatory: true,
+    )
+    ..addFlag(
+      'smoke-test',
+      help: 'runs a single test to verify the setup',
+      negatable: false,
+      defaultsTo: true,
+    );
 
-  final ArgResults results = parser.parse(args);
-  final Directory outDir = Directory(results['out-dir'] as String);
-  final File adb = File(results['adb'] as String);
+  runZonedGuarded(
+    () async {
+      final ArgResults results = parser.parse(args);
+      final Directory outDir = Directory(results['out-dir'] as String);
+      final File adb = File(results['adb'] as String);
+      final bool smokeTest = results['smoke-test'] as bool;
+      await _run(outDir: outDir, adb: adb, smokeTest: smokeTest);
+      exit(0);
+    },
+    (Object error, StackTrace stackTrace) {
+      if (error is! Panic) {
+        stderr.writeln(error);
+        stderr.writeln(stackTrace);
+      }
+      exit(1);
+    },
+  );
+}
+
+Future<void> _run({
+  required Directory outDir,
+  required File adb,
+  required bool smokeTest,
+}) async {
+  const ProcessManager pm = LocalProcessManager();
 
   if (!outDir.existsSync()) {
     panic(<String>['out-dir does not exist: $outDir', 'make sure to build the selected engine variant']);
@@ -170,15 +205,25 @@ void main(List<String> args) async {
     });
 
     await step('Running instrumented tests...', () async {
-      final int exitCode = await pm.runAndForward(<String>[
+      final (int exitCode, StringBuffer out) = await pm.runAndCapture(<String>[
         adb.path,
         'shell',
         'am',
         'instrument',
-        '-w', 'dev.flutter.scenarios.test/dev.flutter.TestRunner',
+        '-w',
+        if (smokeTest)
+          '-e class dev.flutter.scenarios.EngineLaunchE2ETest',
+        'dev.flutter.scenarios.test/dev.flutter.TestRunner',
       ]);
       if (exitCode != 0) {
-        panic(<String>['could not install test apk']);
+        panic(<String>['instrumented tests failed to run']);
+      }
+      // Unfortunately adb shell am instrument does not return a non-zero exit
+      // code when tests fail, but it does seem to print "FAILURES!!!" to
+      // stdout, so we can use that as a signal that something went wrong.
+      if (out.toString().contains('FAILURES!!!')) {
+        stdout.write(out);
+        panic(<String>['1 or more tests failed']);
       }
     });
   } finally {
@@ -221,8 +266,7 @@ void main(List<String> args) async {
 
     await step('Flush logcat...', () async {
       await logcat.flush();
+      await logcat.close();
     });
-
-    exit(0);
   }
 }

testing/scenario_app/bin/utils/logs.dart

@@ -23,9 +23,11 @@ void log(String msg) {
   stdout.writeln('$_gray$msg$_reset');
 }
 
+final class Panic extends Error {}
+
 void panic(List<String> messages) {
   for (final String message in messages) {
     stderr.writeln('$_red$message$_reset');
   }
-  throw 'panic';
+  throw Panic();
 }

testing/scenario_app/bin/utils/process_manager_extension.dart

@@ -45,4 +45,11 @@ extension RunAndForward on ProcessManager {
   Future<int> runAndForward(List<String> cmd) async {
     return pipeProcessStreams(await start(cmd), out: stdout);
   }
+
+  /// Runs [cmd], and captures the stdout and stderr pipes.
+  Future<(int, StringBuffer)> runAndCapture(List<String> cmd) async {
+    final StringBuffer buffer = StringBuffer();
+    final int exitCode = await pipeProcessStreams(await start(cmd), out: buffer);
+    return (exitCode, buffer);
+  }
 }

testing/scenario_app/lib/main.dart

@@ -25,14 +25,19 @@ void main() {
   channelBuffers.setListener('driver', _handleDriverMessage);
   channelBuffers.setListener('write_timeline', _handleWriteTimelineMessage);
 
-  final FlutterView view = PlatformDispatcher.instance.implicitView!;
+  // TODO(matanlurey): https://github.com/flutter/flutter/issues/142746.
+  // This Dart program is used for every test, but there is at least one test
+  // (EngineLaunchE2ETest.java) that does not create a FlutterView, so the
+  // implicit view's size is not initialized (and the assert would be tripped).
+  //
+  // final FlutterView view = PlatformDispatcher.instance.implicitView!;
   // Asserting that this is greater than zero since this app runs on different
   // platforms with different sizes. If it is greater than zero, it has been
   // initialized to some meaningful value at least.
-  assert(
-    view.display.size > Offset.zero,
-    'Expected ${view.display} to be initialized.',
-  );
+  // assert(
+  //   view.display.size > Offset.zero,
+  //   'Expected ${view.display} to be initialized.',
+  // );
 
   final ByteData data = ByteData(1);
   data.setUint8(0, 1);

testing/scenario_app/run_android_tests.sh

@@ -33,7 +33,10 @@ function follow_links() (
 )
 
 SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
-SRC_DIR="$(cd "$SCRIPT_DIR/../../.."; pwd -P)"
+SRC_DIR="$(
+  cd "$SCRIPT_DIR/../../.."
+  pwd -P
+)"
 OUT_DIR="$SRC_DIR/out/$BUILD_VARIANT"
 
 # Dump the logcat and symbolize stack traces before exiting.

testing/scenario_app/tool/run_android_tests_smoke.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This is a debugging script that runs a single Android E2E test on a connected
+# device or emulator, and reports the exit code. It was largely created to debug
+# why `./testing/scenario_app/run_android_tests.sh` did or did not report
+# failures correctly.
+
+# Run this command and print out the exit code.
+../third_party/dart/tools/sdks/dart-sdk/bin/dart ./testing/scenario_app/bin/android_integration_tests.dart \
+  --adb="../third_party/android_tools/sdk/platform-tools/adb" \
+  --out-dir="../out/android_debug_unopt_arm64" \
+  --smoke-test
+
+echo "Exit code: $?"
+echo "Done"