Metrics: Add a new e2e/metrics folder to start writing some test to c…

…ollect performance metrics

packages/calypso-e2e/src/lib/pages/editor-page.ts

@@ -204,6 +204,30 @@ export class EditorPage {
 
 	//#region Basic Entry
 
+	/**
+	 * Loads HTML into the editor.
+	 * @param content Content to load.
+	 */
+	async loadHtmlContent( content: string ) {
+		const editorParent = await this.getEditorParent();
+		await editorParent.evaluate( ( _, html ) => {
+			// @ts-expect-error Untyped global variable.
+			const { parse } = window.wp.blocks;
+			// @ts-expect-error Untyped global variable.
+			const { dispatch } = window.wp.data;
+			const blocks = parse( html );
+
+			blocks.forEach( ( block: any ) => {
+				if ( block.name === 'core/image' ) {
+					delete block.attributes.id;
+					delete block.attributes.url;
+				}
+			} );
+
+			dispatch( 'core/block-editor' ).resetBlocks( blocks );
+		}, content );
+	}
+
 	/**
 	 * Selects blank template from the template modal.
 	 */

packages/calypso-e2e/src/lib/utils/index.ts

@@ -2,6 +2,7 @@
 export * from './validate-translations';
 export * from './get-test-account-by-feature';
 export * from './translate';
+export * from './metrics';
 
 // Other items are exported for unit testing, we only care about the manager class.
 export { EditorTracksEventManager } from './editor-tracks-event-manager';

packages/calypso-e2e/src/lib/utils/metrics.ts

@@ -0,0 +1,234 @@
+/**
+ * External dependencies
+ */
+import type { Page, Browser } from 'playwright';
+
+type EventType =
+	| 'click'
+	| 'focus'
+	| 'focusin'
+	| 'keydown'
+	| 'keypress'
+	| 'keyup'
+	| 'mouseout'
+	| 'mouseover';
+
+interface TraceEvent {
+	cat: string;
+	name: string;
+	dur?: number;
+	args: {
+		data?: {
+			type: EventType;
+		};
+	};
+}
+
+interface Trace {
+	traceEvents: TraceEvent[];
+}
+
+interface WebVitalsMeasurements {
+	CLS?: number;
+	FCP?: number;
+	FID?: number;
+	INP?: number;
+	LCP?: number;
+	TTFB?: number;
+}
+
+/**
+ * Metrics class to collect performance metrics.
+ */
+export class Metrics {
+	browser: Browser;
+	page: Page;
+	trace: Trace;
+
+	webVitals: WebVitalsMeasurements = {};
+
+	/**
+	 * @param page Playwright page object.
+	 */
+	constructor( page: Page ) {
+		this.page = page;
+		this.browser = page.context().browser()!;
+		this.trace = { traceEvents: [] };
+	}
+
+	/**
+	 * Returns durations from the Server-Timing header.
+	 *
+	 * @param fields Optional fields to filter.
+	 */
+	async getServerTiming( fields: string[] = [] ) {
+		return this.page.evaluate< Record< string, number >, string[] >(
+			( f: string[] ) =>
+				(
+					performance.getEntriesByType( 'navigation' ) as PerformanceNavigationTiming[]
+				 )[ 0 ].serverTiming.reduce(
+					( acc, entry ) => {
+						if ( f.length === 0 || f.includes( entry.name ) ) {
+							acc[ entry.name ] = entry.duration;
+						}
+						return acc;
+					},
+					{} as Record< string, number >
+				),
+			fields
+		);
+	}
+
+	/**
+	 * Returns time to first byte (TTFB) using the Navigation Timing API.
+	 *
+	 * @see https://web.dev/ttfb/#measure-ttfb-in-javascript
+	 *
+	 * @returns TTFB value.
+	 */
+	async getTimeToFirstByte() {
+		return await this.page.evaluate< number >( () => {
+			const { responseStart, startTime } = (
+				performance.getEntriesByType( 'navigation' ) as PerformanceNavigationTiming[]
+			 )[ 0 ];
+			return responseStart - startTime;
+		} );
+	}
+
+	/**
+	 * Returns the Largest Contentful Paint (LCP) value using the dedicated API.
+	 *
+	 * @see https://w3c.github.io/largest-contentful-paint/
+	 * @see https://web.dev/lcp/#measure-lcp-in-javascript
+	 *
+	 * @returns LCP value.
+	 */
+	async getLargestContentfulPaint() {
+		return await this.page.evaluate< number >(
+			() =>
+				new Promise( ( resolve ) => {
+					new PerformanceObserver( ( entryList ) => {
+						const entries = entryList.getEntries();
+						// The last entry is the largest contentful paint.
+						const largestPaintEntry = entries.at( -1 );
+
+						resolve( largestPaintEntry?.startTime || 0 );
+					} ).observe( {
+						type: 'largest-contentful-paint',
+						buffered: true,
+					} );
+				} )
+		);
+	}
+
+	/**
+	 * Returns the loading durations using the Navigation Timing API. All the
+	 * durations exclude the server response time.
+	 *
+	 * @returns Object with loading metrics durations.
+	 */
+	async getLoadingDurations() {
+		return await this.page.evaluate( () => {
+			const [
+				{ requestStart, responseStart, responseEnd, domContentLoadedEventEnd, loadEventEnd },
+			] = performance.getEntriesByType( 'navigation' ) as PerformanceNavigationTiming[];
+			const paintTimings = performance.getEntriesByType( 'paint' ) as PerformancePaintTiming[];
+
+			const firstPaintStartTime = paintTimings.find(
+				( { name } ) => name === 'first-paint'
+			)!.startTime;
+
+			const firstContentfulPaintStartTime = paintTimings.find(
+				( { name } ) => name === 'first-contentful-paint'
+			)!.startTime;
+
+			return {
+				// Server side metric.
+				serverResponse: responseStart - requestStart,
+				// For client side metrics, consider the end of the response (the
+				// browser receives the HTML) as the start time (0).
+				firstPaint: firstPaintStartTime - responseEnd,
+				domContentLoaded: domContentLoadedEventEnd - responseEnd,
+				loaded: loadEventEnd - responseEnd,
+				firstContentfulPaint: firstContentfulPaintStartTime - responseEnd,
+				timeSinceResponseEnd: performance.now() - responseEnd,
+			};
+		} );
+	}
+
+	/**
+	 * Starts Chromium tracing with predefined options for performance testing.
+	 *
+	 * @param options Options to pass to `browser.startTracing()`.
+	 */
+	async startTracing( options = {} ) {
+		return await this.browser.startTracing( this.page, {
+			screenshots: false,
+			categories: [ 'devtools.timeline' ],
+			...options,
+		} );
+	}
+
+	/**
+	 * Stops Chromium tracing and saves the trace.
+	 */
+	async stopTracing() {
+		const traceBuffer = await this.browser.stopTracing();
+		const traceJSON = JSON.parse( traceBuffer.toString() );
+
+		this.trace = traceJSON;
+	}
+
+	/**
+	 * @returns Durations of all traced `keydown`, `keypress`, and `keyup`
+	 * events.
+	 */
+	getTypingEventDurations() {
+		return [
+			this.getEventDurations( 'keydown' ),
+			this.getEventDurations( 'keypress' ),
+			this.getEventDurations( 'keyup' ),
+		];
+	}
+
+	/**
+	 * @returns Durations of all traced `focus` and `focusin` events.
+	 */
+	getSelectionEventDurations() {
+		return [ this.getEventDurations( 'focus' ), this.getEventDurations( 'focusin' ) ];
+	}
+
+	/**
+	 * @returns Durations of all traced `click` events.
+	 */
+	getClickEventDurations() {
+		return [ this.getEventDurations( 'click' ) ];
+	}
+
+	/**
+	 * @returns Durations of all traced `mouseover` and `mouseout` events.
+	 */
+	getHoverEventDurations() {
+		return [ this.getEventDurations( 'mouseover' ), this.getEventDurations( 'mouseout' ) ];
+	}
+
+	/**
+	 * @param eventType Type of event to filter.
+	 * @returns Durations of all events of a given type.
+	 */
+	getEventDurations( eventType: EventType ) {
+		if ( this.trace.traceEvents.length === 0 ) {
+			throw new Error( 'No trace events found. Did you forget to call stopTracing()?' );
+		}
+
+		return this.trace.traceEvents
+			.filter(
+				( item: TraceEvent ): boolean =>
+					item.cat === 'devtools.timeline' &&
+					item.name === 'EventDispatch' &&
+					item?.args?.data?.type === eventType &&
+					!! item.dur
+			)
+			.map( ( item ) => ( item.dur ? item.dur / 1000 : 0 ) );
+	}
+}

test/e2e/jest.metrics.config.js

@@ -0,0 +1,10 @@
+const baseConfig = require( '@automattic/calypso-e2e/src/jest-playwright-config' );
+
+module.exports = {
+	...baseConfig,
+	cacheDirectory: '<rootDir>/../../.cache/jest',
+	testMatch: [ '<rootDir>/metrics/**/*.[jt]s' ],
+	transform: {
+		'\\.[jt]sx?$': [ 'babel-jest', { configFile: '../../babel.config.js' } ],
+	},
+};

test/e2e/metrics/editor.spec.ts

@@ -0,0 +1,84 @@
+import { readFileSync } from 'fs';
+import path from 'path';
+import {
+	DataHelper,
+	envVariables,
+	EditorPage,
+	TestAccount,
+	getTestAccountByFeature,
+	envToFeatureKey,
+	PostsPage,
+	Metrics,
+} from '@automattic/calypso-e2e';
+import { Browser, Page } from 'playwright';
+
+declare const browser: Browser;
+const results: Record< string, number[] > = {};
+
+describe( DataHelper.createSuiteTitle( 'Metrics: Editor' ), function () {
+	const features = envToFeatureKey( envVariables );
+	const accountName = getTestAccountByFeature(
+		features,
+		// The default accounts for gutenberg+simple are `gutenbergSimpleSiteEdgeUser` for GB edge
+		// and `gutenbergSimpleSiteUser` for stable. The criteria below conflicts with the default
+		// one that would return the `gutenbergSimpleSiteUser`. We also can't define it as part of
+		// the default criteria, and should pass it here, as an override. For this specific function
+		// call, `simpleSitePersonalPlanUser` will be retured when gutenberg is stable, and siteType
+		// is simple.
+		[ { gutenberg: 'stable', siteType: 'simple', accountName: 'simpleSitePersonalPlanUser' } ]
+	);
+
+	let page: Page;
+	let editorPage: EditorPage;
+	let postsPage: PostsPage;
+
+	beforeAll( async () => {
+		page = await browser.newPage();
+
+		const testAccount = new TestAccount( accountName );
+		await testAccount.authenticate( page );
+	} );
+
+	it( 'Start and fill a test post', async function () {
+		postsPage = new PostsPage( page );
+		const metrics = new Metrics( page );
+		await postsPage.visit();
+		await postsPage.newPost();
+		editorPage = new EditorPage( page );
+		await editorPage.waitUntilLoaded();
+		const filePath = path.join( __dirname, './fixtures/large-post.html' );
+		await editorPage.loadHtmlContent( readFileSync( filePath, 'utf8' ).trim() );
+		const canvas = await editorPage.getEditorCanvas();
+		await canvas.locator( '.wp-block' ).first().waitFor();
+		await editorPage.enterTitle( 'Test Post' );
+		await editorPage.publish();
+
+		const samples = 2;
+		const throwaway = 1;
+		const iterations = samples + throwaway;
+
+		for ( let i = 1; i <= iterations; i++ ) {
+			await page.reload();
+			editorPage = new EditorPage( page );
+			editorPage.waitUntilLoaded();
+			const canvas = await editorPage.getEditorCanvas();
+			// Wait for the first block.
+			await canvas.locator( '.wp-block' ).first().waitFor();
+			// Get the durations.
+			const loadingDurations = await metrics.getLoadingDurations();
+
+			// Save the results.
+			if ( i > throwaway ) {
+				Object.entries( loadingDurations ).forEach( ( [ metric, duration ] ) => {
+					const metricKey = metric === 'timeSinceResponseEnd' ? 'firstBlock' : metric;
+					if ( ! results[ metricKey ] ) {
+						results[ metricKey ] = [];
+					}
+					results[ metricKey ].push( duration );
+				} );
+			}
+		}
+
+		console.log( results );
+	} );
+} );