Fix some issues in performance tests (woocommerce#47735)

* Close tour when it is open * Use different matrix * Fix performance include * Add changelog * Fix lint * Add metrics to editor tests
Neosate · May 27, 2024 · 721da06 · 721da06
1 parent 890fd87
commit 721da06
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 34 deletions.
diff --git a/plugins/woocommerce/changelog/fix-performance_metrics_results b/plugins/woocommerce/changelog/fix-performance_metrics_results
@@ -0,0 +1,4 @@
+Significance: minor
+Type: fix
+
+Fix some issues in performance tests #47735
diff --git a/plugins/woocommerce/tests/metrics/config/performance-reporter.ts b/plugins/woocommerce/tests/metrics/config/performance-reporter.ts
@@ -1,3 +1,7 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/**
+ * External dependencies
+ */
 import path from 'path';
 import { writeFileSync } from 'fs';
 import type {
@@ -6,6 +10,7 @@ import type {
 	TestCase,
 	TestResult,
 } from '@playwright/test/reporter';
+/* eslint-enable @typescript-eslint/no-unused-vars */
 
 export type WPPerformanceResults = Record< string, number >;
 
@@ -30,7 +35,9 @@ class PerformanceReporter implements Reporter {
 			const resultsId = process.env.RESULTS_ID || testSuite;
 			const resultsPath = process.env.WP_ARTIFACTS_PATH as string;
 			const resultsBody = attachment.body.toString();
-			const results = JSON.parse( resultsBody );
+			const resultsObj = JSON.parse( resultsBody );
+			const firstKey = Object.keys( resultsObj )[ 0 ];
+			const results = resultsObj[ firstKey ];
 
 			// Save curated results to file.
 			writeFileSync(
@@ -45,7 +52,7 @@ class PerformanceReporter implements Reporter {
 		}
 	}
 
-	onEnd( result: FullResult ) {
+	onEnd( result: FullResult ): void {
 		if ( result.status !== 'passed' ) {
 			return;
 		}

diff --git a/plugins/woocommerce/tests/metrics/specs/editor.spec.js b/plugins/woocommerce/tests/metrics/specs/editor.spec.js
@@ -9,7 +9,7 @@ import { test, Metrics } from '@wordpress/e2e-test-utils-playwright';
  * Internal dependencies
  */
 import { PerfUtils } from '../fixtures';
-import { median } from '../utils';
+import { getTotalBlockingTime, median } from '../utils';
 
 // See https://github.com/WordPress/gutenberg/issues/51383#issuecomment-1613460429
 const BROWSER_IDLE_WAIT = 1000;
@@ -57,11 +57,11 @@ test.describe( 'Editor Performance', () => {
 
 	test.afterAll( async ( {}, testInfo ) => {
 		const medians = {};
-		Object.keys( results ).map( ( metric ) => {
+		Object.keys( results ).forEach( ( metric ) => {
 			medians[ metric ] = median( results[ metric ] );
 		} );
 		await testInfo.attach( 'results', {
-			body: JSON.stringify( medians, null, 2 ),
+			body: JSON.stringify( { editor: medians }, null, 2 ),
 			contentType: 'application/json',
 		} );
 	} );
@@ -95,8 +95,32 @@ test.describe( 'Editor Performance', () => {
 				// Get the durations.
 				const loadingDurations = await metrics.getLoadingDurations();
 
+				// Measure CLS
+				const cumulativeLayoutShift =
+					await metrics.getCumulativeLayoutShift();
+
+				// Measure LCP
+				const largestContentfulPaint =
+					await metrics.getLargestContentfulPaint();
+
+				// Measure TBT
+				const totalBlockingTime = await getTotalBlockingTime(
+					page,
+					BROWSER_IDLE_WAIT
+				);
+
 				// Save the results.
 				if ( i > throwaway ) {
+					results.totalBlockingTime = results.tbt || [];
+					results.totalBlockingTime.push( totalBlockingTime );
+					results.cumulativeLayoutShift =
+						results.cumulativeLayoutShift || [];
+					results.cumulativeLayoutShift.push( cumulativeLayoutShift );
+					results.largestContentfulPaint =
+						results.largestContentfulPaint || [];
+					results.largestContentfulPaint.push(
+						largestContentfulPaint
+					);
 					Object.entries( loadingDurations ).forEach(
 						( [ metric, duration ] ) => {
 							const metricKey =

diff --git a/plugins/woocommerce/tests/metrics/specs/product-editor.spec.js b/plugins/woocommerce/tests/metrics/specs/product-editor.spec.js
@@ -8,7 +8,7 @@ import { test, Metrics } from '@wordpress/e2e-test-utils-playwright';
 /**
  * Internal dependencies
  */
-import { median } from '../utils';
+import { getTotalBlockingTime, median } from '../utils';
 import { toggleBlockProductEditor } from '../../e2e-pw/utils/simple-products';
 
 // See https://github.com/WordPress/gutenberg/issues/51383#issuecomment-1613460429
@@ -23,31 +23,6 @@ const results = {
 	type: [],
 };
 
-async function getTotalBlockingTime( page, idleWait ) {
-	const totalBlockingTime = await page.evaluate( async ( waitTime ) => {
-		return new Promise( ( resolve ) => {
-			const longTaskEntries = [];
-			// Create a performance observer to observe long task entries
-			new PerformanceObserver( ( list ) => {
-				const entries = list.getEntries();
-				// Store each long task entry in the longTaskEntries array
-				entries.forEach( ( entry ) => longTaskEntries.push( entry ) );
-			} ).observe( { type: 'longtask', buffered: true } );
-
-			// Give some time to collect entries
-			setTimeout( () => {
-				// Calculate the total blocking time by summing the durations of all long tasks
-				const tbt = longTaskEntries.reduce(
-					( acc, entry ) => acc + entry.duration,
-					0
-				);
-				resolve( tbt );
-			}, waitTime );
-		} );
-	}, idleWait );
-	return totalBlockingTime;
-}
-
 test.describe( 'Product editor performance', () => {
 	test.use( {
 		metrics: async ( { page }, use ) => {
@@ -61,11 +36,11 @@ test.describe( 'Product editor performance', () => {
 
 	test.afterAll( async ( {}, testInfo ) => {
 		const medians = {};
-		Object.keys( results ).map( ( metric ) => {
+		Object.keys( results ).forEach( ( metric ) => {
 			medians[ metric ] = median( results[ metric ] );
 		} );
 		await testInfo.attach( 'results', {
-			body: JSON.stringify( medians, null, 2 ),
+			body: JSON.stringify( { 'product-editor': medians }, null, 2 ),
 			contentType: 'application/json',
 		} );
 	} );
@@ -143,6 +118,15 @@ test.describe( 'Product editor performance', () => {
 				.first()
 				.waitFor();
 
+			// FTUX tour on first run through
+			try {
+				await page
+					.getByLabel( 'Close Tour' )
+					.click( { timeout: 3000 } );
+			} catch ( e ) {
+				console.log( 'Tour was not visible, skipping.' );
+			}
+
 			const input = page.getByPlaceholder( 'e.g. 12 oz Coffee Mug' );
 
 			// The first character typed triggers a longer time (isTyping change).

diff --git a/plugins/woocommerce/tests/metrics/utils.js b/plugins/woocommerce/tests/metrics/utils.js
@@ -22,3 +22,28 @@ export function readFile( filePath ) {
 
 	return readFileSync( filePath, 'utf8' ).trim();
 }
+
+export async function getTotalBlockingTime( page, idleWait ) {
+	const totalBlockingTime = await page.evaluate( async ( waitTime ) => {
+		return new Promise( ( resolve ) => {
+			const longTaskEntries = [];
+			// Create a performance observer to observe long task entries
+			new PerformanceObserver( ( list ) => {
+				const entries = list.getEntries();
+				// Store each long task entry in the longTaskEntries array
+				entries.forEach( ( entry ) => longTaskEntries.push( entry ) );
+			} ).observe( { type: 'longtask', buffered: true } );
+
+			// Give some time to collect entries
+			setTimeout( () => {
+				// Calculate the total blocking time by summing the durations of all long tasks
+				const tbt = longTaskEntries.reduce(
+					( acc, entry ) => acc + entry.duration,
+					0
+				);
+				resolve( tbt );
+			}, waitTime );
+		} );
+	}, idleWait );
+	return totalBlockingTime;
+}
diff --git a/tools/compare-perf/performance.js b/tools/compare-perf/performance.js
@@ -368,7 +368,7 @@ async function runPerformanceTests( branches, options ) {
 			const resultsRounds = resultFiles
 				.filter( ( file ) =>
 					file.includes(
-						`${ testSuite }_${ sanitizedBranchName }_round-`
+						`/${ testSuite }_${ sanitizedBranchName }_round-`
 					)
 				)
 				.map( ( file ) => {