Fix snapshot tests in CI (#6156)

* Add snapshot tests job to PR's and master build.
* Add record option to snapshot tests job.
* Archive failing snapshot tests as artifacts.

Co-authored-by: wixmobile <mobile1@wix.com>
Co-authored-by: Guy Carmeli <guyca@users.noreply.github.com>

playground/.gitignore

@@ -209,3 +209,6 @@ buck-out/
 \.buckd/
 android/app/libs
 android/keystores/debug.keystore
+
+# Snapshot tests diffs
+ios/SnapshotTests/FailureDiffs/

playground/ios/Podfile.lock

@@ -221,7 +221,7 @@ PODS:
     - ReactCommon/jscallinvoker (= 0.61.5)
   - ReactNativeKeyboardTrackingView (5.6.1):
     - React
-  - ReactNativeNavigation (6.3.1):
+  - ReactNativeNavigation (6.4.0):
     - React
     - React-RCTImage
     - React-RCTText
@@ -352,9 +352,9 @@ SPEC CHECKSUMS:
   React-RCTVibration: a49a1f42bf8f5acf1c3e297097517c6b3af377ad
   ReactCommon: 198c7c8d3591f975e5431bec1b0b3b581aa1c5dd
   ReactNativeKeyboardTrackingView: a240a6a0dba852bb107109a7ec7e98b884055977
-  ReactNativeNavigation: d9bb71088ef37aa6af58f3172f890a1ae08861d6
+  ReactNativeNavigation: 667586f6924fbd09512e2933fe70497db914b758
   Yoga: f2a7cd4280bfe2cca5a7aed98ba0eb3d1310f18b
 
-PODFILE CHECKSUM: 480983e790dd4e9f99b1673fbb65606d9fe8ce71
+PODFILE CHECKSUM: 51b8040756c3fb06cf4c6437b0a7c683703c2b03
 
 COCOAPODS: 1.9.1

playground/ios/SnapshotTests/StackOptionsTest.m

@@ -16,8 +16,9 @@ - (void)setUp {
 	_commandsHandler = [CommandsHandlerCreator createWithWindow:_window];
 	self.usesDrawViewHierarchyInRect = YES;
 
-//	Uncomment next line to record new snapshots
-//	self.recordMode = YES;
+	if (NSProcessInfo.processInfo.environment[@"RECORD_SNAPSHOTS"]) {
+		self.recordMode = YES;
+	}
 }
 
 - (void)tearDown {

playground/ios/playground.xcodeproj/project.pbxproj

@@ -1091,7 +1091,7 @@
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				GCC_C_LANGUAGE_STANDARD = gnu11;
 				INFOPLIST_FILE = SnapshotTests/Info.plist;
-				IPHONEOS_DEPLOYMENT_TARGET = 13.2;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
@@ -1118,7 +1118,7 @@
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				GCC_C_LANGUAGE_STANDARD = gnu11;
 				INFOPLIST_FILE = SnapshotTests/Info.plist;
-				IPHONEOS_DEPLOYMENT_TARGET = 13.2;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = rn.SnapshotTests;

playground/ios/playground.xcodeproj/xcshareddata/xcschemes/SnapshotRecordTests.xcscheme

@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1120"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "NO">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "507C80D92429111F00F765F7"
+            BuildableName = "SnapshotTests.xctest"
+            BlueprintName = "SnapshotTests"
+            ReferencedContainer = "container:playground.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+      <EnvironmentVariables>
+         <EnvironmentVariable
+            key = "FB_REFERENCE_IMAGE_DIR"
+            value = "$(SOURCE_ROOT)/SnapshotTests/ReferenceImages"
+            isEnabled = "YES">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "RECORD_SNAPSHOTS"
+            value = "YES"
+            isEnabled = "YES">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "IMAGE_DIFF_DIR"
+            value = "$(SOURCE_ROOT)/SnapshotTests/FailureDiffs"
+            isEnabled = "YES">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "TEST_ENABLED"
+            value = "YES"
+            isEnabled = "YES">
+         </EnvironmentVariable>
+      </EnvironmentVariables>
+      <Testables>
+         <TestableReference
+            skipped = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "507C80D92429111F00F765F7"
+               BuildableName = "SnapshotTests.xctest"
+               BlueprintName = "SnapshotTests"
+               ReferencedContainer = "container:playground.xcodeproj">
+            </BuildableReference>
+         </TestableReference>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "13B07F861A680F5B00A75B9A"
+            BuildableName = "playground.app"
+            BlueprintName = "playground"
+            ReferencedContainer = "container:playground.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+      <EnvironmentVariables>
+         <EnvironmentVariable
+            key = "RECORD_SNAPSHOT_TESTS"
+            value = ""
+            isEnabled = "YES">
+         </EnvironmentVariable>
+      </EnvironmentVariables>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "13B07F861A680F5B00A75B9A"
+            BuildableName = "playground.app"
+            BlueprintName = "playground"
+            ReferencedContainer = "container:playground.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

playground/ios/playground.xcodeproj/xcshareddata/xcschemes/SnapshotTests.xcscheme

@@ -26,6 +26,11 @@
             value = "$(SOURCE_ROOT)/SnapshotTests/ReferenceImages"
             isEnabled = "YES">
          </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "IMAGE_DIFF_DIR"
+            value = "$(SOURCE_ROOT)/SnapshotTests/FailureDiffs"
+            isEnabled = "YES">
+         </EnvironmentVariable>
          <EnvironmentVariable
             key = "TEST_ENABLED"
             value = "YES"

scripts/test-snapshot.js

@@ -3,6 +3,8 @@ const exec = require('shell-utils').exec;
 
 const android = includes(process.argv, '--android');
 const release = includes(process.argv, '--release');
+const BRANCH = process.env.BRANCH;
+const RECORD = process.env.RECORD === 'true';
 
 function run() {
   if (android) {
@@ -19,7 +21,7 @@ function runAndroidSnapshotTests() {
 function runIosSnapshotTests() {
   exec.execSync('npm run build');
   exec.execSync('npm run pod-install');
-  testTarget('SnapshotTests', 'iPhone 11');
+  testTarget(RECORD ? 'SnapshotRecordTests' : 'SnapshotTests', 'iPhone 11', '13.0');
 }
 
 function testTarget(scheme, device, OS = 'latest') {
@@ -36,16 +38,43 @@ function testTarget(scheme, device, OS = 'latest') {
   -UseModernBuildSystem=NO
   ONLY_ACTIVE_ARCH=YES`);
 
-  exec.execSync(`cd ./playground/ios &&
-  RCT_NO_LAUNCH_PACKAGER=true
-  xcodebuild test-without-building
-  -scheme "${scheme}"
-  -workspace playground.xcworkspace
-  -sdk iphonesimulator
-  -configuration ${conf}
-  -destination 'platform=iOS Simulator,name=${device},OS=${OS}'
-  -derivedDataPath ./DerivedData/playground
-  ONLY_ACTIVE_ARCH=YES`);
+  try {
+    exec.execSync(`cd ./playground/ios &&
+    RCT_NO_LAUNCH_PACKAGER=true
+    xcodebuild test-without-building
+    -scheme "${scheme}"
+    -workspace playground.xcworkspace
+    -sdk iphonesimulator
+    -configuration ${conf}
+    -destination 'platform=iOS Simulator,name=${device},OS=${OS}'
+    -derivedDataPath ./DerivedData/playground
+    ONLY_ACTIVE_ARCH=YES`);
+  } catch (error) {
+    if (!RECORD) {
+      throw 'Snapshot tests failed';
+    }
+  }
+  finally {
+    if (RECORD) {
+      pushSnapshots();
+    }
+  }
+}
+
+function pushSnapshots() {
+  setupGit();
+  exec.execSync(`git checkout ${BRANCH}`);
+  exec.execSync(`git add ./playground/ios/SnapshotTests/ReferenceImages_64`);
+  exec.execSync(`git commit -m "Update snapshots [ci skip]"`);
+  exec.execSync(`git push deploy ${BRANCH}`);
+}
+
+function setupGit() {
+  exec.execSyncSilent(`git config --global push.default simple`);
+  exec.execSyncSilent(`git config --global user.email "${process.env.GIT_EMAIL}"`);
+  exec.execSyncSilent(`git config --global user.name "${process.env.GIT_USER}"`);
+  const remoteUrl = new RegExp(`https?://(\\S+)`).exec(exec.execSyncRead(`git remote -v`))[1];
+  exec.execSyncSilent(`git remote add deploy "https://${process.env.GIT_USER}:${process.env.GIT_TOKEN}@${remoteUrl}"`);
 }
 
 run();