fix(test-runner-visual-regression): allow mismatched screenshot sizes to be diffed

.changeset/few-sloths-grin.md

@@ -0,0 +1,6 @@
+---
+"@web/test-runner-visual-regression": patch
+"@web/test-runner": patch
+---
+
+Capture visual regressions across changing screenshot sizes.

packages/test-runner-visual-regression/src/config.ts

@@ -28,6 +28,7 @@ export type OptionalImage = Buffer | undefined | Promise<Buffer | undefined>;
 export interface DiffResult {
   diffPercentage: number;
   diffImage: Buffer;
+  error: string;
 }
 
 export interface DiffArgs {

packages/test-runner-visual-regression/src/pixelMatchDiff.ts

@@ -1,30 +1,36 @@
 import pixelmatch from 'pixelmatch';
-import { PNG } from 'pngjs';
+import { PNG, PNGWithMetadata } from 'pngjs';
 
 import { DiffArgs, DiffResult } from './config';
-import { VisualRegressionError } from './VisualRegressionError';
 
 export function pixelMatchDiff({ baselineImage, image, options }: DiffArgs): DiffResult {
-  const basePng = PNG.sync.read(baselineImage);
-  const png = PNG.sync.read(image);
+  let error = '';
+  let basePng: PNG | PNGWithMetadata = PNG.sync.read(baselineImage);
+  let png: PNG | PNGWithMetadata = PNG.sync.read(image);
+  let { width, height } = png;
 
   if (basePng.width !== png.width || basePng.height !== png.height) {
-    throw new VisualRegressionError(
+    error =
       `Screenshot is not the same width and height as the baseline. ` +
-        `Baseline: { width: ${basePng.width}, height: ${basePng.height} }` +
-        `Screenshot: { width: ${png.width}, height: ${png.height} }`,
-    );
+      `Baseline: { width: ${basePng.width}, height: ${basePng.height} }` +
+      `Screenshot: { width: ${png.width}, height: ${png.height} }`;
+    width = Math.max(basePng.width, png.width);
+    height = Math.max(basePng.height, png.height);
+    let oldPng = basePng;
+    basePng = new PNG({ width, height });
+    oldPng.data.copy(basePng.data, 0, 0, oldPng.data.length);
+    oldPng = png;
+    png = new PNG({ width, height });
+    oldPng.data.copy(png.data, 0, 0, oldPng.data.length);
   }
 
-  const width = png.width;
-  const height = png.height;
-
   const diff = new PNG({ width, height });
 
   const numDiffPixels = pixelmatch(basePng.data, png.data, diff.data, width, height, options);
   const diffPercentage = (numDiffPixels / (width * height)) * 100;
 
   return {
+    error,
     diffImage: PNG.sync.write(diff),
     diffPercentage,
   };

packages/test-runner-visual-regression/src/visualDiffCommand.ts

@@ -1,6 +1,7 @@
 import path from 'path';
 
 import { VisualRegressionPluginOptions } from './config';
+import { VisualRegressionError } from './VisualRegressionError';
 
 function resolveImagePath(baseDir: string, name: string) {
   const finalName = path.extname(name) ? name : `${name}.png`;
@@ -53,6 +54,15 @@ export async function visualDiffCommand(
     });
   };
 
+  const saveDiff = async () => {
+    await options.saveDiff({
+      filePath: resolveImagePath(baseDir, diffName),
+      baseDir,
+      name: diffName,
+      content: diffImage,
+    });
+  };
+
   if (!baselineImage) {
     await saveFailed();
 
@@ -63,21 +73,25 @@ export async function visualDiffCommand(
     };
   }
 
-  const { diffImage, diffPercentage } = await options.getImageDiff({
+  const { diffImage, diffPercentage, error } = await options.getImageDiff({
     name,
     baselineImage,
     image,
     options: options.diffOptions,
   });
+
+  if (error) {
+    // The diff has failed, be sure to save the new image.
+    await saveFailed();
+    await saveDiff();
+
+    throw new VisualRegressionError(error);
+  }
+
   const passed = diffPercentage === 0;
 
   if (!passed) {
-    await options.saveDiff({
-      filePath: resolveImagePath(baseDir, diffName),
-      baseDir,
-      name: diffName,
-      content: diffImage,
-    });
+    await saveDiff();
   }
 
   if (!passed || options.buildCache) {