Introduce parser for dynamic token system

packages/tokens/test/index.js

@@ -0,0 +1,32 @@
+/**
+ * External dependencies
+ */
+import path from 'path';
+
+/**
+ * Internal dependencies
+ */
+import { swapTokens } from '../token-parser';
+import { jsTester, phpTester } from './shared.test';
+
+const parse = ( input ) => {
+	let count = 0;
+	const tokens = [];
+
+	const tokenReplacer = ( token ) => {
+		tokens.push( token );
+		return `{{TOKEN_${ ++count }}}`;
+	};
+
+	const output = swapTokens(
+		'https://token.wordpress.org/',
+		tokenReplacer,
+		input
+	);
+
+	return { tokens, output };
+};
+
+describe( 'tokens', jsTester( parse ) ); // eslint-disable-line jest/valid-describe-callback
+
+phpTester( 'token-parser-php', path.join( __dirname, 'test-parser.php' ) );

packages/tokens/test/shared.test.js

@@ -0,0 +1,129 @@
+export const jsTester = ( parse ) => () => {
+	describe( 'various forms of the token syntax', () => {
+		it( 'recognizes basic forms of the token syntax', () => {
+			expect(
+				parse(
+					'https://token.wordpress.org/#token{"name":"core/identity"}#'
+				)
+			).toEqual( {
+				tokens: [
+					{
+						namespace: 'core',
+						name: 'identity',
+						attributes: {},
+						fallback: '',
+					},
+				],
+				output: '{{TOKEN_1}}',
+			} );
+
+			expect(
+				parse( 'https://token.wordpress.org/#token{core/identity}#' )
+			).toEqual( {
+				namespace: 'core',
+				name: 'identity',
+				attributes: {},
+				fallback: '',
+			} );
+
+			expect(
+				parse(
+					'https://token.wordpress.org/#token{core/echo="\u{3c}"}#'
+				)
+			).toEqual( {
+				namespace: 'core',
+				name: 'echo',
+				attributes: { value: '<' },
+				fallback: '',
+			} );
+
+			expect(
+				parse(
+					'https://token.wordpress.org/#token{{"name":"my_plugin/widget", "attributes": {"name": "sprocket"}, "fallback": "just a sprocket"}#'
+				)
+			).toEqual( {
+				namespace: 'my_plugin',
+				name: 'widget',
+				attributes: { name: 'sprocket' },
+				fallback: 'just a sprocket',
+			} );
+
+			expect(
+				parse(
+					'https://token.wordpress.org/#token{my_plugin/widget,{"name":"sprocket"},"just a sprocket"}#'
+				)
+			).toEqual( {
+				namespace: 'my_plugin',
+				name: 'widget',
+				attributes: { name: 'sprocket' },
+				fallback: 'just a sprocket',
+			} );
+		} );
+	} );
+};
+
+const hasPHP =
+	'test' === process.env.NODE_ENV
+		? ( () => {
+				const process = require( 'child_process' ).spawnSync(
+					'php',
+					[ '-r', 'echo 1;' ],
+					{
+						encoding: 'utf8',
+					}
+				);
+
+				return process.status === 0 && process.stdout === '1';
+		  } )()
+		: false;
+
+// Skipping if `php` isn't available to us, such as in local dev without it
+// skipping preserves snapshots while commenting out or simply
+// not injecting the tests prompts `jest` to remove "obsolete snapshots"
+const makeTest = hasPHP
+	? // eslint-disable-next-line jest/valid-describe-callback, jest/valid-title
+	  ( ...args ) => describe( ...args )
+	: // eslint-disable-next-line jest/no-disabled-tests, jest/valid-describe-callback, jest/valid-title
+	  ( ...args ) => describe.skip( ...args );
+
+export const phpTester = ( name, filename ) =>
+	makeTest(
+		name,
+		'test' === process.env.NODE_ENV
+			? jsTester( ( doc ) => {
+					const process = require( 'child_process' ).spawnSync(
+						'php',
+						[ '-f', filename ],
+						{
+							input: doc,
+							encoding: 'utf8',
+							timeout: 30 * 1000, // Abort after 30 seconds, that's too long anyway.
+						}
+					);
+
+					if ( process.status !== 0 ) {
+						throw new Error( process.stderr || process.stdout );
+					}
+
+					try {
+						/*
+						 * Due to an issue with PHP's json_encode() serializing an empty associative array
+						 * as an empty list `[]` we're manually replacing the already-encoded bit here.
+						 *
+						 * This is an issue with the test runner, not with the parser.
+						 */
+						return JSON.parse(
+							process.stdout.replace(
+								/"attributes":\s*\[\]/g,
+								'"attributes":{}'
+							)
+						);
+					} catch ( e ) {
+						console.error( process.stdout );
+						throw new Error(
+							'failed to parse JSON:\n' + process.stdout
+						);
+					}
+			  } )
+			: () => {}
+	);

packages/tokens/test/test-parser.php

@@ -0,0 +1,25 @@
+<?php
+/**
+ * PHP Test Helper
+ *
+ * Facilitates running PHP parser against same tests as the JS parser implementation.
+ *
+ * @package gutenberg
+ */
+
+// Include the generated parser.
+require_once __DIR__ . '/../token-parser.php';
+
+$count = 0;
+$tokens = [];
+
+$output = WP_Token_Parser::swap_tokens(
+	'https://token.wordpress.org/',
+	function ( $token ) use ( &$count, &$tokens ) {
+		$tokens[] = $token;
+		return '{{TOKEN_' . ++$count . '}}';
+	},
+	file_get_contents( 'php://stdin' )
+);
+
+echo json_encode( [ 'tokens' => $tokens, 'output' => $output ] );

packages/tokens/token-parser.js

@@ -0,0 +1,84 @@
+/**
+ * @typedef {Object} WPToken
+ * @property {string}              namespace  e.g. "core", "query", or "my-plugin"
+ * @property {string}              name       e.g. "home_url", "featured-iamge", "my-token"
+ * @property {Record<string, any>} attributes defined by each token separately; `value` is special.
+ * @property {string}              fallback   what to render if no matching token plugin available.
+ */
+
+/**
+ * @callback TokenReplacer
+ * @param {WPToken} token The parsed token to replace.
+ * @return {string} the replacement string.
+ */
+
+/**
+ * Replaces dynamic tokens in a document with the return value
+ * from their callback or with a fallback value if one exists.
+ *
+ * @param {string}        urlPrefix
+ * @param {TokenReplacer} tokenReplacer
+ * @param {string}        input
+ * @return {string}       output
+ */
+export const swapTokens = ( urlPrefix, tokenReplacer, input ) => {
+	const quotedUrlPrefix = escapeRegExp( urlPrefix );
+
+	return input.replace(
+		new RegExp( `${ quotedUrlPrefix }#token{([^#]*)}#` ),
+		( fullMatch, tokenContents ) => {
+			const token = parseTokenContents( tokenContents );
+			if ( ! token ) {
+				return fullMatch;
+			}
+
+			return tokenReplacer( token ) ?? token.fallback;
+		}
+	);
+};
+
+/**
+ * Parses the inner contents of a token.
+ *
+ * @param {string} contents the inner contents of a token to parse.
+ * @return {WPToken|null} the parsed token or null if invalid.
+ */
+const parseTokenContents = ( contents ) => {
+	const matches = contents.match(
+		/^([a-z][a-z\d-]*)\/([a-z\d-]*)(?:=(.+))?$/i
+	);
+	if ( matches ) {
+		const [ , namespace, name, rawValue ] = matches;
+		const value = rawValue ? jsonDecode( rawValue ) : null;
+
+		return value
+			? { namespace, name, attributes: { value }, fallback: '' }
+			: { namespace, name, attributes: {}, fallback: '' };
+	}
+
+	const tokenData = jsonDecode( `{${ contents }}` );
+	if ( null === tokenData ) {
+		return null;
+	}
+
+	const nameMatch = tokenData.name?.match(
+		/^([a-z][a-z\d-]*)\/([a-z\d-]*)(?:=(.+))?$/i
+	);
+	if ( ! nameMatch ) {
+		return null;
+	}
+
+	const [ , namespace, name ] = nameMatch;
+
+	return { attributes: {}, fallback: '', ...tokenData, namespace, name };
+};
+
+const jsonDecode = ( s ) => JSON.parse( s );
+
+/**
+ * Borrowed directly from lodash to avoid including dependency.
+ *
+ * @param {string} raw input which might contain meaningful RegExp syntax.
+ * @return {string} input with meaningful RegExp syntax escaped.
+ */
+const escapeRegExp = ( raw ) => raw.replace( /[\\^$.*+?()[\]{}|]/g, '\\$&' );