[RFC] feat(test-runner-visual-regression): make "update" strict and add "buildCache" for more repeatable tests

.changeset/violet-gifts-turn.md

@@ -0,0 +1,8 @@
+---
+"@web/test-runner-core": patch
+"@web/test-runner-visual-regression": minor
+"@web/test-runner": patch
+---
+
+Add `buildCache` option to the visual regression config to support always saving the "current" screenshot.
+Make the `update` option in the visual regression config _strict_, and only save "current" shots as "baseline" when it is set to `true`.

packages/test-runner-core/package.json

@@ -54,8 +54,8 @@
     "@types/co-body": "^5.1.0",
     "@types/convert-source-map": "^1.5.1",
     "@types/debounce": "^1.2.0",
-    "@types/istanbul-reports": "^3.0.0",
     "@types/istanbul-lib-coverage": "^2.0.3",
+    "@types/istanbul-reports": "^3.0.0",
     "@types/uuid": "^8.3.0",
     "@web/browser-logs": "^0.2.1",
     "@web/dev-server-core": "^0.3.6",

packages/test-runner-visual-regression/src/config.ts

@@ -43,6 +43,12 @@ export interface VisualRegressionPluginOptions {
    * the image with the current baseline.
    */
   update: boolean;
+  /**
+   * Whether to build a new cache separate from the baseline
+   * of the new/failed images. Useful for removed and/or
+   * purposely updated tests.
+   */
+  buildCache: boolean;
   /**
    * The base directory to write images to.
    */
@@ -107,6 +113,7 @@ async function saveImage({ filePath, content }: SaveImageArgs) {
 
 export const defaultOptions: VisualRegressionPluginOptions = {
   update: false,
+  buildCache: false,
   baseDir: 'screenshots',
   diffOptions: {},
 

packages/test-runner-visual-regression/src/visualDiffCommand.ts

@@ -31,7 +31,7 @@ export async function visualDiffCommand(
     name: baselineName,
   });
 
-  if (!baselineImage || options.update) {
+  if (options.update) {
     await options.saveBaseline({
       filePath: resolveImagePath(baseDir, baselineName),
       baseDir,
@@ -44,6 +44,25 @@ export async function visualDiffCommand(
   const diffName = options.getDiffName({ browser, name });
   const failedName = options.getFailedName({ browser, name });
 
+  const saveFailed = async () => {
+    await options.saveFailed({
+      filePath: resolveImagePath(baseDir, failedName),
+      baseDir,
+      name: failedName,
+      content: image,
+    });
+  };
+
+  if (!baselineImage) {
+    await saveFailed();
+
+    return {
+      errorMessage: 'There was no baseline image to compare against.',
+      diffPercentage: -1,
+      passed: false,
+    };
+  }
+
   const { diffImage, diffPercentage } = await options.getImageDiff({
     name,
     baselineImage,
@@ -59,13 +78,10 @@ export async function visualDiffCommand(
       name: diffName,
       content: diffImage,
     });
+  }
 
-    await options.saveFailed({
-      filePath: resolveImagePath(baseDir, failedName),
-      baseDir,
-      name: failedName,
-      content: image,
-    });
+  if (!passed || options.buildCache) {
+    await saveFailed();
   }
 
   return {