Content Parsers: Update parsers, fix list-item parsing, fix broken ma…

…nifest (#247)

* Update parsers

This includes the updates from WordPress/wordpress.org#90, a new AttributeParser for attribute-only blocks, and a fix for the ListItem block to allow child lists.
Fixes #211

* Add phpunit test infrastructure and tests

Pulled from #181

* Remove swag page from manifest

* Update content with new parser

* Add a phpunit workflow

.github/workflows/phpunit.yml

@@ -0,0 +1,34 @@
+name: PHP Unit Tests
+
+# This workflow is triggered on pushes to trunk, and any PRs.
+on:
+    push:
+        branches: [trunk]
+    pull_request:
+
+jobs:
+    check:
+        name: Test PHP
+
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
+
+            - name: Install NodeJS
+              uses: actions/setup-node@eeb10cff27034e7acf239c5d29f62154018672fd # v3.3.0
+              with:
+                  node-version-file: '.nvmrc'
+                  cache: yarn
+
+            - name: Install dependencies & setup configs
+              run: |
+                  yarn setup:tools
+
+            - name: Install WordPress
+              run: |
+                  yarn wp-env start
+
+            - name: Running PHP unit tests
+              run: yarn test:php

.gitignore

@@ -23,6 +23,7 @@ phpcs.xml.dist
 /source/wp-content/mu-plugins/*
 
 # ...except the actual source of this project.
+!/source/wp-content/tests
 !/source/wp-content/themes
 !/source/wp-content/themes/wporg-main-2022
 !/source/wp-content/mu-plugins

.wp-env.json

@@ -28,5 +28,16 @@
 		"wp-content/mu-plugins": "./source/wp-content/mu-plugins",
 		"wp-content/mu-plugins/0-sandbox.php": "./env/0-sandbox.php",
 		"wp-cli.yml": "./wp-cli.yml"
+	},
+	"env": {
+		"tests": {
+			"config": {
+				"WP_ENVIRONMENT_TYPE": false
+			},
+			"mappings": {
+				"wp-content/tests": "./source/wp-content/tests",
+				"vendor": "./vendor"
+			}
+		}
 	}
 }

composer.json

@@ -100,7 +100,8 @@
 		"wpackagist-plugin/wordpress-importer": "*",
 		"wporg/wporg-repo-tools": "dev-trunk",
 		"wporg/wporg-mu-plugins": "dev-build",
-		"wporg/wporg-parent-2021": "dev-build"
+		"wporg/wporg-parent-2021": "dev-build",
+		"yoast/phpunit-polyfills": "^1.0"
 	},
 	"scripts": {
 		"format": "phpcbf -p",

env/export-content/includes/parser.php

@@ -3,54 +3,120 @@
 namespace WordPress_org\Main_2022\ExportToPatterns;
 
 require_once __DIR__ . '/parsers/BlockParser.php';
+require_once __DIR__ . '/parsers/AttributeParser.php';
 require_once __DIR__ . '/parsers/BasicText.php';
 require_once __DIR__ . '/parsers/Button.php';
-require_once __DIR__ . '/parsers/Heading.php';
+require_once __DIR__ . '/parsers/HTMLParser.php';
 require_once __DIR__ . '/parsers/ListItem.php';
 require_once __DIR__ . '/parsers/Noop.php';
-require_once __DIR__ . '/parsers/Paragraph.php';
-
-// Unused.
 require_once __DIR__ . '/parsers/ShortcodeBlock.php';
 require_once __DIR__ . '/parsers/TextNode.php';
 
 class BlockParser {
-	public $pattern;
+	public $content;
 	public $parsers = [];
 	public $fallback;
 
-	public function __construct() {
-		$this->parsers = [
-			// Core blocks that have custom parsers.
-			'core/paragraph'   => new Parsers\Paragraph(),
+	public function __construct( string $content = '' ) {
+		$this->content  = $content;
+		$this->fallback = new Parsers\BasicText();
+		$this->parsers  = [
+			// Blocks that have custom parsers.
+			'core/paragraph'   => new Parsers\HTMLParser( 'p' ),
+			'core/image'       => new Parsers\HTMLParser( 'figcaption', [ 'alt', 'title' ] ),
+			'core/heading'     => new Parsers\HTMLRegexParser( '/h[1-6]/' ),
+
 			'core/list-item'   => new Parsers\ListItem(),
-			'core/heading'     => new Parsers\Heading(),
-			'core/button'      => new Parsers\Button(),
+			//'core/button'      => new Parsers\Button(),
+			//'core/buttons'     => new Parsers\BasicText(),
+			'core/button'      => new Parsers\HTMLParser( 'a', [ 'title' ] ),
+
+			// Attributes handler.
+			'core/navigation-link' => new Parsers\AttributeParser( [ 'label' ] ),
+			'core/social-link'     => new Parsers\AttributeParser( [ 'label' ] ),
+
+			// Generic shortcode handler.
+			'core/shortcode'   => new Parsers\ShortcodeBlock(),
+
 			'core/spacer'      => new Parsers\Noop(),
+			// These contain other blocks to be parsed.
+			'core/column'      => new Parsers\Noop(),
+			'core/columns'     => new Parsers\Noop(),
+			'core/group'       => new Parsers\Noop(),
+			'core/list'        => new Parsers\Noop(),
+			'core/quote'       => new Parsers\Noop(),
 
 			// Common core blocks that use the default parser.
-			'core/buttons'     => new Parsers\BasicText(),
-			'core/list'        => new Parsers\BasicText(),
-			'core/column'      => new Parsers\BasicText(),
-			'core/columns'     => new Parsers\BasicText(),
-			'core/cover'       => new Parsers\BasicText(),
-			'core/group'       => new Parsers\BasicText(),
-			'core/image'       => new Parsers\BasicText(),
 			'core/media-text'  => new Parsers\BasicText(),
-			'core/separator'   => new Parsers\BasicText(),
-			'core/social-link' => new Parsers\BasicText(),
 		];
+	}
 
-		$this->fallback = new Parsers\BasicText();
+	public static function post_to_strings( $post ) {
+		// TODO: Detect post using a block template, pull strings from there.
+		$self    = new self( $post->post_content );
+		$strings = $self->to_strings();
+
+		if ( $post->post_title ) {
+			$strings[] = $post->post_title;
+		}
+		if ( $post->post_excerpt ) {
+			$strings[] = $post->post_excerpt;
+		}
+
+		$post_meta_to_include = apply_filters( 'translatable_post_meta', [] );
+		foreach ( $post_meta_to_include as $meta_key ) {
+			$strings[] = get_post_meta( $post->ID, $meta_key, true );
+		}
+
+		return $strings;
+	}
+
+	public static function translate_post( $post, callable $callback_translate ) {
+		$post->post_content = self::translate_blocks( $post->post_content, $callback_translate ) ?: $post->post_content;
+		$post->post_title   = $callback_translate( $post->post_title ) ?: $post->post_title;
+		$post->post_excerpt = $callback_translate( $post->post_excerpt ) ?: $post->post_excerpt;
+
+		return $post;
 	}
 
-	public function replace_with_i18n( string $content ) : string {
-		$strings = $this->to_strings( $content );
-		$i18n_strings = [];
+	public static function translate_blocks( string $content, callable $callback_translate ) /*: bool|string*/ {
+		$self         = new self( $content );
+
+		$translations = [];
+		$translated   = false;
+		$strings      = $self->to_strings();
+
 		foreach ( $strings as $string ) {
-			$i18n_strings[ $string ] = sprintf( "<?php _e( '%s', 'wporg' ); ?>", str_replace( "'", '&#039;', $string ) );
+			$translations[ $string ] = $callback_translate( $string );
+
+			$translated = $translated || ( $string !== $translations[ $string ] );
 		}
-		return $this->replace_strings( $content, $i18n_strings );
+
+		// Are there any translations?
+		if ( ! $translated ) {
+			return false;
+		}
+
+		return $self->replace_strings_with_kses( $translations );
+	}
+
+	public static function translate_block( string $content, $block, callable $callback_translate ) /* :bool|string */ {
+		$self    = new self();
+		$parser  = $self->parsers[ $block['blockName'] ] ?? $self->fallback;
+		$strings = $parser->to_strings( $block ); // does not do innerBlocks, intentionally.
+
+		if ( ! $strings ) {
+			return $content;
+		}
+
+		$replacements = [];
+		foreach ( $strings as $string ) {
+			$replacements[ $string ] = $callback_translate( $string ) ?: $string;
+		}
+
+		$block = $parser->replace_strings( $block, $replacements );
+
+		return $block['innerContent'][0] ?: $content;
 	}
 
 	public function block_parser_to_strings( array $block ) : array {
@@ -62,7 +128,7 @@ public function block_parser_to_strings( array $block ) : array {
 			$strings = array_merge( $strings, $this->block_parser_to_strings( $inner_block ) );
 		}
 
-		return $strings;
+		return array_unique( $strings );
 	}
 
 	public function block_parser_replace_strings( array &$block, array $replacements ) : array {
@@ -76,21 +142,31 @@ public function block_parser_replace_strings( array &$block, array $replacements
 		return $block;
 	}
 
-	public function to_strings( string $content ) : array {
-		$blocks = parse_blocks( $content );
-
+	public function to_strings() : array {
 		$strings = [];
 
+		$blocks = parse_blocks( $this->content );
+
 		foreach ( $blocks as $block ) {
 			$strings = array_merge( $strings, $this->block_parser_to_strings( $block ) );
 		}
 
 		return array_unique( $strings );
 	}
 
-	public function replace_strings( string $content, array $replacements ) : string {
-		$blocks = parse_blocks( $content );
+	public function replace_strings_with_kses( array $replacements ) : string {
+		// Sanitize replacement strings before injecting them into blocks and block attributes.
+		$sanitized_replacements = $replacements;
+		foreach ( $sanitized_replacements as &$replacement ) {
+			$replacement = wp_kses_post( $replacement );
+		}
+		return $this->replace_strings( $sanitized_replacements );
+	}
+
+	public function replace_strings( array $replacements ) : string {
+		$translated = $this->content;
 
+		$blocks = parse_blocks( $translated );
 		foreach ( $blocks as &$block ) {
 			$block = $this->block_parser_replace_strings( $block, $replacements );
 		}
@@ -100,9 +176,9 @@ public function replace_strings( string $content, array $replacements ) : string
 		// "subscribePlaceholder":"😀" becomes "subscribePlaceholder":"\ud83d\ude00".
 		// After we get the serialized blocks back from `serialize_blocks` we need to convert these
 		// characters back to their unicode form so that we don't break blocks in the editor.
-		$new_content = $this->decode_unicode_characters( serialize_blocks( $blocks ) );
+		$translated = $this->decode_unicode_characters( serialize_blocks( $blocks ) );
 
-		return $new_content;
+		return $translated;
 	}
 
 	/**
@@ -144,6 +220,28 @@ function ( $matches ) use ( $excluded_characters ) {
 			$string
 		);
 
+		// Decode < & > if they're part of a PHP tag.
+		$decoded_string = str_replace( [ '\\u003c?', '?\\u003e' ], [ '<?', '?>' ], $decoded_string );
+
 		return $decoded_string;
 	}
 }
+
+/**
+ * Helper function to replace all strings in content with i18n-wrapped strings.
+ */
+function replace_with_i18n( string $content, string $textdomain = 'wporg' ) : string {
+	$parser = new BlockParser( $content );
+	$strings = $parser->to_strings();
+
+	$i18n_strings = [];
+	foreach ( $strings as $string ) {
+		$i18n_strings[ $string ] = sprintf(
+			"<?php _e( '%s', '%s' ); ?>",
+			str_replace( "'", '&#039;', $string ),
+			$textdomain
+		);
+	}
+
+	return $parser->replace_strings( $i18n_strings );
+}