[Impeller] Skia gold for flutter_tester dart tests. (#47066)

This removes skips for the golden tests in `//testing/dart/canvas_test.dart` and instead passes them up to Skia gold.

Adds a utility class for dealing with Skia gold from these tests, as well as the existing fuzzy identical image comparison for tests that just want to do in memory comparison of images generated from the same test.

Removes the old golden files that were in tree.

Part of flutter/flutter#53784

Author: dnfield

.ci.yaml

@@ -293,6 +293,10 @@ targets:
       add_recipes_cq: "true"
       release_build: "true"
       config_name: linux_host_engine
+      dependencies: >-
+        [
+          {"dependency": "goldctl", "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"}
+        ]
     drone_dimensions:
       - os=Linux
 

ci/builders/linux_host_engine.json

@@ -185,6 +185,12 @@
                 "device_type=none",
                 "os=Linux"
             ],
+            "dependencies": [
+                {
+                    "dependency": "goldctl",
+                    "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"
+                }
+            ],
             "gclient_variables": {
                 "download_android_deps": false
             },

impeller/golden_tests_harvester/bin/golden_tests_harvester.dart

@@ -16,13 +16,16 @@ bool get isLuciEnv => Platform.environment.containsKey(_kLuciEnvName);
 
 /// Fake SkiaGoldClient that is used if the harvester is run outside of Luci.
 class FakeSkiaGoldClient implements SkiaGoldClient {
-  FakeSkiaGoldClient(this._workingDirectory, {this.dimensions});
+  FakeSkiaGoldClient(this._workingDirectory, {this.dimensions, this.verbose = false});
 
   final Directory _workingDirectory;
 
   @override
   final Map<String, String>? dimensions;
 
+  @override
+  final bool verbose;
+
   @override
   Future<void> addImg(String testName, File goldenFile,
       {double differentPixelsRate = 0.01,

impeller/golden_tests_harvester/pubspec.yaml

@@ -30,6 +30,8 @@ dependency_overrides:
     path: ../../../third_party/dart/third_party/pkg/file/packages/file
   meta:
     path: ../../../third_party/dart/pkg/meta
+  engine_repo_tools:
+    path: ../../tools/pkg/engine_repo_tools
   path:
     path: ../../../third_party/dart/third_party/pkg/path
   platform:

lib/web_ui/pubspec.yaml

@@ -54,3 +54,7 @@ dev_dependencies:
     path: ../../web_sdk/web_engine_tester
   skia_gold_client:
     path: ../../testing/skia_gold_client
+
+dependency_overrides:
+  engine_repo_tools:
+    path: ../../tools/pkg/engine_repo_tools

testing/dart/BUILD.gn

@@ -48,9 +48,11 @@ tests = [
 ]
 
 foreach(test, tests) {
+  skia_gold_work_dir = rebase_path("$root_gen_dir/skia_gold_$test")
   flutter_frontend_server("compile_$test") {
     main_dart = test
     kernel_output = "$root_gen_dir/$test.dill"
+    extra_args = [ "-DkSkiaGoldWorkDirectory=$skia_gold_work_dir" ]
     package_config = ".dart_tool/package_config.json"
   }
 }

testing/dart/canvas_test.dart

@@ -12,6 +12,7 @@ import 'package:litetest/litetest.dart';
 import 'package:path/path.dart' as path;
 import 'package:vector_math/vector_math_64.dart';
 
+import 'goldens.dart';
 import 'impeller_enabled.dart';
 
 typedef CanvasCallback = void Function(Canvas canvas);
@@ -123,59 +124,9 @@ void testNoCrashes() {
   });
 }
 
-/// @returns true When the images are reasonably similar.
-/// @todo Make the search actually fuzzy to a certain degree.
-Future<bool> fuzzyCompareImages(Image golden, Image img) async {
-  if (golden.width != img.width || golden.height != img.height) {
-    return false;
-  }
-  int getPixel(ByteData data, int x, int y) => data.getUint32((x + y * golden.width) * 4);
-  final ByteData goldenData = (await golden.toByteData())!;
-  final ByteData imgData = (await img.toByteData())!;
-  for (int y = 0; y < golden.height; y++) {
-    for (int x = 0; x < golden.width; x++) {
-      if (getPixel(goldenData, x, y) != getPixel(imgData, x, y)) {
-        return false;
-      }
-    }
-  }
-  return true;
-}
-
-Future<void> saveTestImage(Image image, String filename) async {
-  final String imagesPath = path.join('flutter', 'testing', 'resources');
-  final ByteData pngData = (await image.toByteData(format: ImageByteFormat.png))!;
-  final String outPath = path.join(imagesPath, filename);
-  File(outPath).writeAsBytesSync(pngData.buffer.asUint8List());
-  print('wrote: $outPath');
-}
-
-/// @returns true When the images are reasonably similar.
-Future<bool> fuzzyGoldenImageCompare(
-    Image image, String goldenImageName) async {
-  final String imagesPath = path.join('flutter', 'testing', 'resources');
-  final File file = File(path.join(imagesPath, goldenImageName));
-
-  bool areEqual = false;
-
-  if (file.existsSync()) {
-    final Uint8List goldenData = await file.readAsBytes();
-
-    final Codec codec = await instantiateImageCodec(goldenData);
-    final FrameInfo frame = await codec.getNextFrame();
-    expect(frame.image.height, equals(image.height));
-    expect(frame.image.width, equals(image.width));
-
-    areEqual = await fuzzyCompareImages(frame.image, image);
-  }
+void main() async {
+  final ImageComparer comparer = await ImageComparer.create();
 
-  if (!areEqual) {
-    saveTestImage(image, 'found_$goldenImageName');
-  }
-  return areEqual;
-}
-
-void main() {
   testNoCrashes();
 
   test('Simple .toImage', () async {
@@ -190,11 +141,8 @@ void main() {
     }, 100, 100);
     expect(image.width, equals(100));
     expect(image.height, equals(100));
-
-    final bool areEqual =
-        await fuzzyGoldenImageCompare(image, 'canvas_test_toImage.png');
-    expect(areEqual, true);
-  }, skip: impellerEnabled);
+    await comparer.addGoldenImage(image, 'canvas_test_toImage.png');
+  });
 
   Gradient makeGradient() {
     return Gradient.linear(
@@ -212,10 +160,8 @@ void main() {
     expect(image.width, equals(100));
     expect(image.height, equals(100));
 
-    final bool areEqual =
-        await fuzzyGoldenImageCompare(image, 'canvas_test_dithered_gradient.png');
-    expect(areEqual, true);
-  }, skip: !Platform.isLinux || impellerEnabled); // https://github.com/flutter/flutter/issues/53784
+    await comparer.addGoldenImage(image, 'canvas_test_dithered_gradient.png');
+  });
 
   test('Null values allowed for drawAtlas methods', () async {
     final Image image = await createImage(100, 100);
@@ -302,12 +248,8 @@ void main() {
       });
     }, width, height);
 
-    final bool areEqual = await fuzzyCompareImages(incrementalMatrixImage, combinedMatrixImage);
+    final bool areEqual = await comparer.fuzzyCompareImages(incrementalMatrixImage, combinedMatrixImage);
 
-    if (!areEqual) {
-      saveTestImage(incrementalMatrixImage, 'incremental_3D_transform_test_image.png');
-      saveTestImage(combinedMatrixImage, 'combined_3D_transform_test_image.png');
-    }
     expect(areEqual, true);
   });
 
@@ -348,10 +290,8 @@ void main() {
     expect(image.width, equals(200));
     expect(image.height, equals(250));
 
-    final bool areEqual =
-        await fuzzyGoldenImageCompare(image, 'dotted_path_effect_mixed_with_stroked_geometry.png');
-    expect(areEqual, true);
-  }, skip: !Platform.isLinux || impellerEnabled); // https://github.com/flutter/flutter/issues/53784
+    await comparer.addGoldenImage(image, 'dotted_path_effect_mixed_with_stroked_geometry.png');
+  });
 
   test('Gradients with matrices in Paragraphs render correctly', () async {
     final Image image = await toImage((Canvas canvas) {
@@ -400,10 +340,8 @@ void main() {
     expect(image.width, equals(600));
     expect(image.height, equals(400));
 
-    final bool areEqual =
-    await fuzzyGoldenImageCompare(image, 'text_with_gradient_with_matrix.png');
-    expect(areEqual, true);
-  }, skip: !Platform.isLinux || impellerEnabled); // https://github.com/flutter/flutter/issues/53784
+    await comparer.addGoldenImage(image, 'text_with_gradient_with_matrix.png');
+  });
 
   test('toImageSync - too big', () async {
     PictureRecorder recorder = PictureRecorder();
@@ -602,8 +540,8 @@ void main() {
     final Image tofuImage = await drawText('>\b<');
 
     // The tab's image should be identical to the space's image but not the tofu's image.
-    final bool tabToSpaceComparison = await fuzzyCompareImages(tabImage, spaceImage);
-    final bool tabToTofuComparison = await fuzzyCompareImages(tabImage, tofuImage);
+    final bool tabToSpaceComparison = await comparer.fuzzyCompareImages(tabImage, spaceImage);
+    final bool tabToTofuComparison = await comparer.fuzzyCompareImages(tabImage, tofuImage);
 
     expect(tabToSpaceComparison, isTrue);
     expect(tabToTofuComparison, isFalse);

testing/dart/goldens.dart

@@ -0,0 +1,129 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:typed_data';
+import 'dart:ui';
+
+import 'package:path/path.dart' as path;
+import 'package:skia_gold_client/skia_gold_client.dart';
+
+import 'impeller_enabled.dart';
+
+const String _kSkiaGoldWorkDirectoryKey = 'kSkiaGoldWorkDirectory';
+
+/// A helper for doing image comparison (golden) tests.
+///
+/// Contains utilities for comparing two images in memory that are expected to
+/// be identical, or for adding images to Skia gold for comparison.
+class ImageComparer {
+  ImageComparer._({
+    required SkiaGoldClient client,
+  }) : _client = client;
+
+  // Avoid talking to Skia gold for the force-multithreading variants.
+  static bool get _useSkiaGold =>
+      !Platform.executableArguments.contains('--force-multithreading');
+
+  /// Creates an image comparer and authorizes.
+  static Future<ImageComparer> create({
+    bool verbose = false,
+  }) async {
+    const String workDirectoryPath =
+        String.fromEnvironment(_kSkiaGoldWorkDirectoryKey);
+    if (workDirectoryPath.isEmpty) {
+      throw UnsupportedError(
+          'Using ImageComparer requries defining kSkiaGoldWorkDirectoryKey.');
+    }
+
+    final Directory workDirectory = Directory(
+      impellerEnabled ? '${workDirectoryPath}_iplr' : workDirectoryPath,
+    )..createSync();
+    final Map<String, String> dimensions = <String, String>{
+      'impeller_enabled': impellerEnabled.toString(),
+    };
+    final SkiaGoldClient client = isSkiaGoldClientAvailable && _useSkiaGold
+        ? SkiaGoldClient(workDirectory,
+            dimensions: dimensions, verbose: verbose)
+        : _FakeSkiaGoldClient(workDirectory, dimensions, verbose: verbose);
+
+    await client.auth();
+    return ImageComparer._(client: client);
+  }
+
+  final SkiaGoldClient _client;
+
+  /// Adds an [Image] to Skia Gold for comparison.
+  ///
+  /// The [fileName] must be unique.
+  Future<void> addGoldenImage(Image image, String fileName) async {
+    final ByteData data =
+        (await image.toByteData(format: ImageByteFormat.png))!;
+
+    final File file = File(path.join(_client.workDirectory.path, fileName))
+      ..writeAsBytesSync(data.buffer.asUint8List());
+    await _client.addImg(
+      fileName,
+      file,
+      screenshotSize: image.width * image.height,
+    ).catchError((dynamic error) {
+      print('Skia gold comparison failed: $error');
+      throw Exception('Failed comparison: $fileName');
+    });
+  }
+
+  Future<bool> fuzzyCompareImages(Image golden, Image testImage) async {
+    if (golden.width != testImage.width || golden.height != testImage.height) {
+      return false;
+    }
+    int getPixel(ByteData data, int x, int y) =>
+        data.getUint32((x + y * golden.width) * 4);
+    final ByteData goldenData = (await golden.toByteData())!;
+    final ByteData testImageData = (await testImage.toByteData())!;
+    for (int y = 0; y < golden.height; y++) {
+      for (int x = 0; x < golden.width; x++) {
+        if (getPixel(goldenData, x, y) != getPixel(testImageData, x, y)) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+}
+
+// TODO(dnfield): add local comparison against baseline,
+// https://github.com/flutter/flutter/issues/136831
+class _FakeSkiaGoldClient implements SkiaGoldClient {
+  _FakeSkiaGoldClient(
+    this.workDirectory,
+    this.dimensions, {
+    this.verbose = false,
+  });
+
+  @override
+  final Directory workDirectory;
+
+  @override
+  final Map<String, String> dimensions;
+
+  @override
+  final bool verbose;
+
+  @override
+  Future<void> auth() async {}
+
+  @override
+  Future<void> addImg(
+    String testName,
+    File goldenFile, {
+    double differentPixelsRate = 0.01,
+    int pixelColorDelta = 0,
+    required int screenshotSize,
+  }) async {}
+
+  @override
+  dynamic noSuchMethod(Invocation invocation) {
+    throw UnimplementedError(invocation.memberName.toString().split('"')[1]);
+  }
+}

testing/dart/pubspec.yaml

@@ -19,6 +19,7 @@ environment:
 dependencies:
   litetest: any
   path: any
+  skia_gold_client: any
   sky_engine: any
   vector_math: any
   vm_service: any
@@ -29,8 +30,14 @@ dependency_overrides:
     path: ../../../third_party/dart/pkg/async_helper
   collection:
     path: ../../../third_party/dart/third_party/pkg/collection
+  crypto:
+    path: ../../../third_party/dart/third_party/pkg/crypto
+  engine_repo_tools:
+    path: ../../tools/pkg/engine_repo_tools
   expect:
     path: ../../../third_party/dart/pkg/expect
+  file:
+    path: ../../../third_party/dart/third_party/pkg/file/packages/file
   fixnum:
     path: ../../../third_party/dart/third_party/pkg/fixnum
   litetest:
@@ -39,12 +46,20 @@ dependency_overrides:
     path: ../../../third_party/dart/pkg/meta
   path:
     path: ../../../third_party/dart/third_party/pkg/path
+  platform:
+    path: ../../../third_party/pkg/platform
+  process:
+    path: ../../../third_party/pkg/process
   protobuf:
     path: ../../../third_party/dart/third_party/pkg/protobuf/protobuf
   smith:
     path: ../../../third_party/dart/pkg/smith
+  skia_gold_client:
+    path: ../skia_gold_client
   sky_engine:
     path: ../../sky/packages/sky_engine
+  typed_data:
+    path: ../../../third_party/dart/third_party/pkg/typed_data
   vector_math:
     path: ../../../third_party/pkg/vector_math
   vm_service: