adding screenshot capability to text_editing e2e test

Author: nturgut

e2etests/web/regular_integration_tests/lib/screenshot_support.dart

@@ -0,0 +1,78 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+
+import 'package:flutter_driver/flutter_driver.dart';
+import 'package:golden_comparator/goldens.dart';
+import 'package:integration_test/integration_test_driver_extended.dart' as test;
+
+import 'package:golden_comparator/image_compare.dart';
+
+import 'package:image/image.dart';
+
+/// Tolerable pixel difference ratio between the goldens and the screenshots.
+///
+/// We are allowing a higher difference rate compared to the unit tests (where
+/// this rate is set to 0.28), since during the end to end tests there are
+/// more components on the screen which are not related to the functinality
+/// under test ex: a blinking cursor.
+const double kMaxDiffRateFailure = 0.5 / 100; // 0.5%
+
+/// Used for calling `integration_test` package.
+///
+/// Compared to other similar classes which only included the following call:
+/// ```
+/// Future<void> main() async => test.integrationDriver();
+/// ```
+///
+/// this method is able to take screenshot.
+///
+/// It provides an `onScreenshot` callback to the `integrationDriver` method.
+/// It also includes options for updating the golden files.
+Future<void> main() async {
+  final WebFlutterDriver driver =
+      await FlutterDriver.connect() as WebFlutterDriver;
+
+  // Learn the browser in use from the webDriver.
+  final String browser = driver.webDriver.capabilities['browserName'] as String;
+
+  bool updateGoldens = false;
+  try {
+    // We are using an environment variable since instead of an argument, since
+    // this code is not invoked from the shell but from the `flutter drive`
+    // tool itself. Therefore we do not have control on the command line
+    // arguments.
+    // Please read the README, further info on how to update the goldens.
+    updateGoldens =
+        io.Platform.environment['UPDATE_GOLDENS'].toLowerCase() == 'true';
+  } catch (ex) {
+    if (ex
+        .toString()
+        .contains('is not a subtype of type \'bool\' in type cast')) {
+      print('INFO: goldens will not be updated, please set `UPDATE_GOLDENS` '
+          'environment variable to true');
+    }
+  }
+
+  test.integrationDriver(
+    driver: driver,
+    onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
+      final Image screenshot = decodePng(screenshotBytes);
+      final String result = compareImage(
+        screenshot,
+        updateGoldens,
+        '$screenshotName-$browser.png',
+        PixelComparison.fuzzy,
+        kMaxDiffRateFailure,
+        forIntegrationTests: true,
+        write: updateGoldens,
+      );
+      if (result != 'OK') {
+        print('ERROR: $result');
+      }
+      return result == 'OK';
+    },
+  );
+}

e2etests/web/regular_integration_tests/lib/text_editing_main.dart

@@ -56,6 +56,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _emptyController,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Empty Input Field:',
               ),
             ),
@@ -67,6 +68,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _controller,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Text Input Field:',
               ),
             ),
@@ -78,6 +80,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _controller2,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Text Input Field 2:',
               ),
               onFieldSubmitted: (String str) {

e2etests/web/regular_integration_tests/pubspec.yaml

@@ -15,7 +15,8 @@ dev_dependencies:
     sdk: flutter
   integration_test: 0.9.0
   http: 0.12.0+2
-  test: any
+  golden_comparator:
+    path: ../../../web_sdk/golden_comparator
 
 flutter:
   assets:

e2etests/web/regular_integration_tests/test_driver/text_editing_integration.dart

@@ -13,7 +13,7 @@ import 'package:flutter/material.dart';
 import 'package:integration_test/integration_test.dart';
 
 void main() {
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+  final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;
 
   testWidgets('Focused text field creates a native input element',
       (WidgetTester tester) async {
@@ -41,6 +41,8 @@ void main() {
     textFormField.controller.text = 'New Value';
     // DOM element's value also changes.
     expect(input.value, 'New Value');
+
+    await binding.takeScreenshot('focused_text_field');
   });
 
   testWidgets('Input field with no initial value works',

e2etests/web/regular_integration_tests/test_driver/text_editing_integration_test.dart

@@ -2,15 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'package:flutter_driver/flutter_driver.dart';
-import 'package:integration_test/integration_test_driver_extended.dart' as test;
+import 'package:regular_integration_tests/screenshot_support.dart'
+    as with_screenshot;
 
 Future<void> main() async {
-  final FlutterDriver driver = await FlutterDriver.connect();
-  test.integrationDriver(
-    driver: driver,
-    onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
-      return true;
-    },
-  );
+  await with_screenshot.main();
 }

lib/web_ui/dev/test_platform.dart

@@ -21,6 +21,8 @@ import 'package:shelf_static/shelf_static.dart';
 import 'package:shelf_web_socket/shelf_web_socket.dart';
 import 'package:shelf_packages_handler/shelf_packages_handler.dart';
 import 'package:stream_channel/stream_channel.dart';
+import 'package:golden_comparator/goldens.dart';
+import 'package:golden_comparator/image_compare.dart';
 import 'package:web_socket_channel/web_socket_channel.dart';
 
 import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
@@ -39,7 +41,6 @@ import 'package:test_core/src/runner/configuration.dart'; // ignore: implementat
 import 'browser.dart';
 import 'common.dart';
 import 'environment.dart' as env;
-import 'goldens.dart';
 import 'screenshot_manager.dart';
 import 'supported_browsers.dart';
 
@@ -197,7 +198,6 @@ class BrowserPlatform extends PlatformPlugin {
         'golden_files',
       );
     } else {
-      await fetchGoldens();
       goldensDirectory = p.join(
         env.environment.webUiGoldensRepositoryDirectory.path,
         'engine',
@@ -215,128 +215,14 @@ class BrowserPlatform extends PlatformPlugin {
     // Take screenshot.
     final Image screenshot = await _screenshotManager.capture(regionAsRectange);
 
-    // Bail out fast if golden doesn't exist, and user doesn't want to create it.
-    final File file = File(p.join(
-      goldensDirectory,
-      filename,
-    ));
-    if (!file.existsSync() && !write) {
-      return '''
-Golden file $filename does not exist.
-
-To automatically create this file call matchGoldenFile('$filename', write: true).
-''';
-    }
-
-    if (write) {
-      // Don't even bother with the comparison, just write and return
-      print('Updating screenshot golden: $file');
-      file.writeAsBytesSync(encodePng(screenshot), flush: true);
-      if (doUpdateScreenshotGoldens) {
-        // Do not fail tests when bulk-updating screenshot goldens.
-        return 'OK';
-      } else {
-        return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.';
-      }
-    }
-
-    // Compare screenshots.
-    ImageDiff diff = ImageDiff(
-      golden: decodeNamedImage(file.readAsBytesSync(), filename),
-      other: screenshot,
-      pixelComparison: pixelComparison,
-    );
-
-    if (diff.rate > 0) {
-      final String testResultsPath =
-          env.environment.webUiTestResultsDirectory.path;
-      final String basename = p.basenameWithoutExtension(file.path);
-
-      final File actualFile =
-          File(p.join(testResultsPath, '$basename.actual.png'));
-      actualFile.writeAsBytesSync(encodePng(screenshot), flush: true);
-
-      final File diffFile = File(p.join(testResultsPath, '$basename.diff.png'));
-      diffFile.writeAsBytesSync(encodePng(diff.diff), flush: true);
-
-      final File expectedFile =
-          File(p.join(testResultsPath, '$basename.expected.png'));
-      file.copySync(expectedFile.path);
-
-      final File reportFile =
-          File(p.join(testResultsPath, '$basename.report.html'));
-      reportFile.writeAsStringSync('''
-Golden file $filename did not match the image generated by the test.
-
-<table>
-  <tr>
-    <th>Expected</th>
-    <th>Diff</th>
-    <th>Actual</th>
-  </tr>
-  <tr>
-    <td>
-      <img src="$basename.expected.png">
-    </td>
-    <td>
-      <img src="$basename.diff.png">
-    </td>
-    <td>
-      <img src="$basename.actual.png">
-    </td>
-  </tr>
-</table>
-''');
-
-      final StringBuffer message = StringBuffer();
-      message.writeln(
-          'Golden file $filename did not match the image generated by the test.');
-      message.writeln(getPrintableDiffFilesInfo(diff.rate, maxDiffRateFailure));
-      message
-          .writeln('You can view the test report in your browser by opening:');
-
-      // Cirrus cannot serve HTML pages generated by build jobs, so we
-      // archive all the files so that they can be downloaded and inspected
-      // locally.
-      if (isCirrus) {
-        final String taskId = Platform.environment['CIRRUS_TASK_ID'];
-        final String baseArtifactsUrl =
-            'https://api.cirrus-ci.com/v1/artifact/task/$taskId/web_engine_test/test_results';
-        final String cirrusReportUrl = '$baseArtifactsUrl/$basename.report.zip';
-        message.writeln(cirrusReportUrl);
-
-        await Process.run(
-          'zip',
-          <String>[
-            '$basename.report.zip',
-            '$basename.report.html',
-            '$basename.expected.png',
-            '$basename.diff.png',
-            '$basename.actual.png',
-          ],
-          workingDirectory: testResultsPath,
-        );
-      } else {
-        final String localReportPath = '$testResultsPath/$basename.report.html';
-        message.writeln(localReportPath);
-      }
-
-      message.writeln(
-          'To update the golden file call matchGoldenFile(\'$filename\', write: true).');
-      message.writeln('Golden file: ${expectedFile.path}');
-      message.writeln('Actual file: ${actualFile.path}');
-
-      if (diff.rate < maxDiffRateFailure) {
-        // Issue a warning but do not fail the test.
-        print('WARNING:');
-        print(message);
-        return 'OK';
-      } else {
-        // Fail test
-        return '$message';
-      }
-    }
-    return 'OK';
+    return compareImage(
+        screenshot,
+        doUpdateScreenshotGoldens,
+        filename,
+        pixelComparison,
+        maxDiffRateFailure,
+        goldensDirectory: goldensDirectory,
+        write: write);
   }
 
   /// A handler that serves wrapper files used to bootstrap tests.

lib/web_ui/dev/test_runner.dart

@@ -15,6 +15,7 @@ import 'package:test_core/src/runner/hack_register_platform.dart'
 import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_core/src/executable.dart'
     as test; // ignore: implementation_imports
+import 'package:golden_comparator/goldens.dart';
 
 import 'common.dart';
 import 'environment.dart';
@@ -189,6 +190,14 @@ class TestCommand extends Command<bool> with ArgUtils {
     }
     environment.webUiTestResultsDirectory.createSync(recursive: true);
 
+    // If screenshot tests are available, fetch the screenshot goldens.
+    if (isScreenhotTestsAvailable) {
+      final GoldensRepoFetcher goldensRepoFetcher = GoldensRepoFetcher(
+          environment.webUiGoldensRepositoryDirectory,
+          path.join(environment.webUiDevDir.path, 'goldens_lock.yaml'));
+      await goldensRepoFetcher.fetch();
+    }
+
     // In order to run iOS Safari unit tests we need to make sure iOS Simulator
     // is booted.
     if (isSafariIOS) {
@@ -371,14 +380,16 @@ class TestCommand extends Command<bool> with ArgUtils {
       isFirefoxIntegrationTestAvailable ||
       isSafariIntegrationTestAvailable;
 
+  // Whether the tests will do screenshot testing.
   bool get isScreenhotTestsAvailable =>
       isIntegrationTestsAvailable || isUnitTestsScreenshotsAvailable;
 
   // For unit tests screenshot tests and smoke tests only run on:
   // "Chrome/iOS" for LUCI/local.
   bool get isUnitTestsScreenshotsAvailable =>
       ((isChrome && isLuci && io.Platform.isLinux) ||
-          ((isChrome || isSafariIOS) && !isLuci));
+          (isChrome || isSafariIOS) && !isLuci) ||
+      (isSafariIOS && isLuci);
 
   /// Use system flutter instead of cloning the repository.
   ///
@@ -624,7 +635,8 @@ class TestCommand extends Command<bool> with ArgUtils {
 
   /// Runs a batch of tests.
   ///
-  /// Unless [expectFailure] is set to false, sets [io.exitCode] to a non-zero value if any tests fail.
+  /// Unless [expectFailure] is set to false, sets [io.exitCode] to a non-zero
+  /// value if any tests fail.
   Future<void> _runTestBatch(
     List<FilePath> testFiles, {
     @required int concurrency,
@@ -647,7 +659,8 @@ class TestCommand extends Command<bool> with ArgUtils {
       return BrowserPlatform.start(
         browser,
         root: io.Directory.current.path,
-        // It doesn't make sense to update a screenshot for a test that is expected to fail.
+        // It doesn't make sense to update a screenshot for a test that is
+        // expected to fail.
         doUpdateScreenshotGoldens: !expectFailure && doUpdateScreenshotGoldens,
       );
     });

lib/web_ui/pubspec.yaml

@@ -14,7 +14,7 @@ dev_dependencies:
   image: 2.1.13
   js: 0.6.1+1
   mockito: 4.1.1
-  path: 1.7.0
+  path: 1.8.0-nullsafety
   test: 1.14.3
   quiver: 2.1.3
   build_resolvers: 1.3.10
@@ -23,6 +23,10 @@ dev_dependencies:
   build_web_compilers: 2.11.0
   yaml: 2.2.1
   watcher: 0.9.7+15
+  common_test_utils:
+    path: ../../web_sdk/common_test_utils
+  golden_comparator:
+    path: ../../web_sdk/golden_comparator
   web_engine_tester:
     path: ../../web_sdk/web_engine_tester
   simulators: