E2e screenshot tests2 (#21383)

* carrying code

* more changes for carrying the code

* rebase changes onto ios-screenshot tests

* adding screenshot capability to text_editing e2e test

* address some comments

* change enable flag for isUnitTestsScreenshotsAvailable

* addressing the reviewer comments

* change the dependency for path

* add to licencense file

* changing goldens commit no. the new commit has the screenshot goldens

* update readme file

* firefox tests needs LUCI changes

* change to release mode since screenshots were taken in release mode

* change window size

* some argument changes

* small comment change

* test the chrome linux tests again

* use roboto font instead of default font

* addressing reviewer comments

* change commit for goldens

Author: nturgut

ci/licenses_golden/licenses_flutter

@@ -1514,6 +1514,10 @@ FILE: ../../../flutter/vulkan/vulkan_window.cc
 FILE: ../../../flutter/vulkan/vulkan_window.h
 FILE: ../../../flutter/web_sdk/libraries.json
 FILE: ../../../flutter/web_sdk/sdk_rewriter.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/environment.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/exceptions.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/goldens.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/image_compare.dart
 ----------------------------------------------------------------------------------------------------
 Copyright 2013 The Flutter Authors. All rights reserved.
 

e2etests/web/regular_integration_tests/README.md

@@ -37,3 +37,34 @@ flutter drive -v --target=test_driver/text_editing_integration.dart -d web-serve
 ```
 
 More details for "Running Flutter Driver tests with Web" can be found in [wiki](https://github.com/flutter/flutter/wiki/Running-Flutter-Driver-tests-with-Web).
+
+## Adding screenshot tests
+
+In order to test screenshot tests the tests on the driver side needs to call the `integration_test` package with an `onScreenshot` callback which can do a comparison between the `screenshotBytes` taken during the test and a golden file. We added a utility method that can do this comparison by using a golden in `flutter/goldens` repository.
+
+In order to use screenshot testing first, import `screenshot_support.dart` from the driver side test (example: `text_editing_integration_test.dart`). Default value for `diffRateFailure` is 0.5.
+
+```
+import 'package:regular_integration_tests/screenshot_support.dart' as test;
+
+Future<void> main() async {
+  final double kMaxDiffRateFailure = 0.1;
+  await test.runTestWithScreenshots(diffRateFailure = kMaxDiffRateFailure);
+}
+```
+
+In order to run the tests follow these steps:
+
+1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, an update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable.
+
+```
+felt test --integration-tests-only --update-screenshot-goldens
+```
+
+```
+UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt
+```
+
+2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`.
+
+3. Get the commit SHA and replace the `revision` in this file: `engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml`

e2etests/web/regular_integration_tests/fonts/RobotoMono-Bold.ttf

@@ -0,0 +1,96 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+import 'dart:math';
+
+import 'package:flutter_driver/flutter_driver.dart';
+import 'package:integration_test/integration_test_driver_extended.dart' as test;
+
+import 'package:web_test_utils/goldens.dart';
+import 'package:web_test_utils/image_compare.dart';
+import 'package:webdriver/src/async/window.dart';
+
+import 'package:image/image.dart';
+
+/// Tolerable pixel difference ratio between the goldens and the screenshots.
+///
+/// We are allowing a higher difference rate compared to the unit tests (where
+/// this rate is set to 0.28), since during the end to end tests there are
+/// more components on the screen which are not related to the functionality
+/// under test ex: a blinking cursor.
+const double kMaxDiffRateFailure = 0.5 / 100; // 0.5%
+
+/// SBrowser screen dimensions for the Flutter Driver test.
+const int _kScreenshotWidth = 1024;
+const int _kScreenshotHeight = 1024;
+
+/// Used for calling `integration_test` package.
+///
+/// Compared to other similar classes which only included the following call:
+/// ```
+/// Future<void> main() async => test.integrationDriver();
+/// ```
+///
+/// this method is able to take screenshot.
+///
+/// It provides an `onScreenshot` callback to the `integrationDriver` method.
+/// It also includes options for updating the golden files.
+Future<void> runTestWithScreenshots(
+    {double diffRateFailure = kMaxDiffRateFailure,
+    int browserWidth = _kScreenshotWidth,
+    int browserHeight = _kScreenshotHeight}) async {
+  final WebFlutterDriver driver =
+      await FlutterDriver.connect() as WebFlutterDriver;
+
+  // Learn the browser in use from the webDriver.
+  final String browser = driver.webDriver.capabilities['browserName'] as String;
+
+  final Window window = await driver.webDriver.window;
+  window.setSize(Rectangle<int>(0, 0, browserWidth, browserHeight));
+
+  bool updateGoldens = false;
+  // We are using an environment variable instead of an argument, since
+  // this code is not invoked from the shell but from the `flutter drive`
+  // tool itself. Therefore we do not have control on the command line
+  // arguments.
+  // Please read the README, further info on how to update the goldens.
+  final String updateGoldensFlag = io.Platform.environment['UPDATE_GOLDENS'];
+  // Validate if the environment variable is set correctly.
+  if (updateGoldensFlag != null &&
+      !(updateGoldensFlag.toLowerCase() == 'true' ||
+          updateGoldensFlag.toLowerCase() == 'false')) {
+    throw StateError(
+        'UPDATE_GOLDENS environment variable is not set correctly');
+  }
+  if (updateGoldensFlag != null && updateGoldensFlag.toLowerCase() == 'true') {
+    updateGoldens = true;
+  }
+
+  test.integrationDriver(
+    driver: driver,
+    onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
+      if (browser == 'chrome') {
+        final Image screenshot = decodePng(screenshotBytes);
+        final String result = compareImage(
+          screenshot,
+          updateGoldens,
+          '$screenshotName-$browser.png',
+          PixelComparison.fuzzy,
+          diffRateFailure,
+          forIntegrationTests: true,
+          write: updateGoldens,
+        );
+        if (result == 'OK') {
+          return true;
+        } else {
+          io.stderr.writeln('ERROR: $result');
+          return false;
+        }
+      } else {
+        return true;
+      }
+    },
+  );
+}

e2etests/web/regular_integration_tests/fonts/RobotoMono-Regular.ttf

@@ -11,6 +11,7 @@ class MyApp extends StatelessWidget {
   Widget build(BuildContext context) {
     return MaterialApp(
       key: const Key('mainapp'),
+      theme: ThemeData(fontFamily: 'RobotoMono'),
       title: 'Integration Test App',
       home: MyHomePage(title: 'Integration Test App'),
     );
@@ -56,6 +57,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _emptyController,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Empty Input Field:',
               ),
             ),
@@ -67,6 +69,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _controller,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Text Input Field:',
               ),
             ),
@@ -78,6 +81,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _controller2,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Text Input Field 2:',
               ),
               onFieldSubmitted: (String str) {
@@ -94,7 +98,7 @@ class _MyHomePageState extends State<MyHomePage> {
               child: SelectableText(
                 'Lorem ipsum dolor sit amet',
                 key: Key('selectable'),
-                style: TextStyle(fontFamily: 'Roboto', fontSize: 20.0),
+                style: TextStyle(fontFamily: 'RobotoMono', fontSize: 20.0),
               ),
             ),
           ],

e2etests/web/regular_integration_tests/lib/screenshot_support.dart

@@ -15,8 +15,14 @@ dev_dependencies:
     sdk: flutter
   integration_test: 0.9.0
   http: 0.12.0+2
-  test: any
+  web_test_utils:
+    path: ../../../web_sdk/web_test_utils
 
 flutter:
   assets:
     - assets/images/
+  fonts:
+    - family: RobotoMono
+      fonts:
+        - asset: fonts/RobotoMono-Bold.ttf
+        - asset: fonts/RobotoMono-Regular.ttf

e2etests/web/regular_integration_tests/lib/text_editing_main.dart

@@ -13,7 +13,7 @@ import 'package:flutter/material.dart';
 import 'package:integration_test/integration_test.dart';
 
 void main() {
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+  final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;
 
   testWidgets('Focused text field creates a native input element',
       (WidgetTester tester) async {
@@ -41,6 +41,8 @@ void main() {
     textFormField.controller.text = 'New Value';
     // DOM element's value also changes.
     expect(input.value, 'New Value');
+
+    await binding.takeScreenshot('focused_text_field');
   });
 
   testWidgets('Input field with no initial value works',

e2etests/web/regular_integration_tests/pubspec.yaml

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'package:integration_test/integration_test_driver.dart' as test;
+import 'package:regular_integration_tests/screenshot_support.dart' as test;
 
-Future<void> main() async => test.integrationDriver();
+Future<void> main() async {
+  await test.runTestWithScreenshots();
+}

e2etests/web/regular_integration_tests/test_driver/text_editing_integration.dart

@@ -1,2 +1,2 @@
 repository: https://github.com/flutter/goldens.git
-revision: 1a4722227af42c3f51450266016b1a07ae459e73
+revision: da3fef0c0eb849dfbb14b09a088c5f7916677482