Proposal: Support overriding short_open_tag in token_get_all()

Details
-------

Add the flags TOKEN_ENABLE_SHORT_OPEN_TAG/TOKEN_DISABLE_SHORT_OPEN_TAG to
`token_get_all` and `PhpToken::tokenize()`

If both flags are accidentally combined,
TOKEN_DISABLE_SHORT_OPEN_TAG takes precedence.

If neither bit flag is provided, then token_get_all continues to use the value
of the system ini setting `short_open_tag`
- https://www.php.net/manual/en/ini.core.php#ini.short-open-tag

Motivation
----------

- Allow linters/analyzers/IDEs to parse/analyze/lint projects targeting a
  deployment environment supporting short open tags/no short open tags,
  regardless of what the user configured locally.
- Make it more convenient to programmatically convert short open tags to
  standard `<?php` tags in migration scripts
- Allow linters/scripts to easily warn about code with short open tags
  that would be echoed instead of compiled/run (T_INLINE_HTML)
  when short tags are disabled
- In applications/libraries that use token_get_all, avoid platform dependence
  when running the test suites (in PHP versions including this change).
  (And make it convenient to test both setting versions in the same test suite)

Implementation details
----------------------

When preparing to scan tokens from a string or from a file, this copies constant
`CG(short_tags)` corresponding to the system ini setting `short_open_tags`
to `SCNG(short_tags)`.
PHP modules such as `tokenizer` can then conditionally override it
after the call to `zend_prepare_string_for_scanning`
(PECLs can also make use of this safely starting)

To handle edge cases such as compilation triggering an error handler
which calls token_get_all, zend_save_lexical_state and
zend_restore_lexical_state also save/restore `short_tags`

Author: TysonAndre

Zend/zend_globals.h

@@ -323,6 +323,9 @@ struct _zend_php_scanner_globals {
 	int heredoc_indentation;
 	bool heredoc_indentation_uses_spaces;
 
+	/* Short tags - either from defaults or tokenizer extension overrides */
+	bool short_tags;
+
 	/* original (unfiltered) script */
 	unsigned char *script_org;
 	size_t script_org_size;

Zend/zend_language_scanner.h

@@ -57,6 +57,8 @@ typedef struct _zend_lex_state {
 
 	zend_ast *ast;
 	zend_arena *ast_arena;
+
+	bool short_tags;
 } zend_lex_state;
 
 typedef struct _zend_heredoc_label {

Zend/zend_language_scanner.l

@@ -246,6 +246,7 @@ ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state)
 
 	lex_state->on_event = SCNG(on_event);
 	lex_state->on_event_context = SCNG(on_event_context);
+	lex_state->short_tags = SCNG(short_tags);
 
 	lex_state->ast = CG(ast);
 	lex_state->ast_arena = CG(ast_arena);
@@ -289,6 +290,7 @@ ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state)
 
 	SCNG(on_event) = lex_state->on_event;
 	SCNG(on_event_context) = lex_state->on_event_context;
+	SCNG(short_tags) = lex_state->short_tags;
 
 	CG(ast) = lex_state->ast;
 	CG(ast_arena) = lex_state->ast_arena;
@@ -731,6 +733,7 @@ ZEND_API void zend_prepare_string_for_scanning(zval *str, zend_string *filename)
 
 	SCNG(yy_in) = NULL;
 	SCNG(yy_start) = NULL;
+	SCNG(short_tags) = CG(short_tags);
 
 	buf = Z_STRVAL_P(str);
 	size = old_len;
@@ -2239,7 +2242,7 @@ string:
 		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
 	}
 	/* Degenerate case: <?phpX is interpreted as <? phpX with short tags. */
-	if (CG(short_tags)) {
+	if (SCNG(short_tags)) {
 		yyless(2);
 		BEGIN(ST_IN_SCRIPTING);
 		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
@@ -2248,7 +2251,7 @@ string:
 }
 
 <INITIAL>"<?" {
-	if (CG(short_tags)) {
+	if (SCNG(short_tags)) {
 		BEGIN(ST_IN_SCRIPTING);
 		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
 	} else {
@@ -2273,7 +2276,7 @@ inline_char_handler:
 		}
 
 		if (*YYCURSOR == '?') {
-			if (CG(short_tags) /* <? */
+			if (SCNG(short_tags) /* <? */
 				|| (*(YYCURSOR + 1) == '=') /* <?= */
 				|| (!strncasecmp((char*)YYCURSOR + 1, "php", 3) && /* <?php[ \t\r\n] */
 					(YYCURSOR + 4 == YYLIMIT ||

ext/tokenizer/php_tokenizer.h

@@ -24,6 +24,8 @@ extern zend_module_entry tokenizer_module_entry;
 #define PHP_TOKENIZER_VERSION PHP_VERSION
 
 #define TOKEN_PARSE (1 << 0)
+#define TOKEN_ENABLE_SHORT_OPEN_TAG (1 << 1)
+#define TOKEN_DISABLE_SHORT_OPEN_TAG (1 << 2)
 
 #ifdef ZTS
 #include "TSRM.h"

ext/tokenizer/tests/bug81342.phpt

@@ -1,5 +1,7 @@
 --TEST--
 Bug #81342: New ampersand token parsing depends on new line after it
+--EXTENSIONS--
+tokenizer
 --FILE--
 <?php
 

ext/tokenizer/tests/short_open_tag.phpt

@@ -0,0 +1,20 @@
+--TEST--
+token_get_all() TOKEN_*_SHORT_OPEN_TAG flag
+--EXTENSIONS--
+tokenizer
+--FILE--
+<?php
+// This can be overridden for individual calls to token_get_all
+echo json_encode(token_get_all('<?xml', TOKEN_DISABLE_SHORT_OPEN_TAG)), "\n";
+echo json_encode(token_get_all('<?xml', TOKEN_ENABLE_SHORT_OPEN_TAG)), "\n";
+// When setting both flags, TOKEN_DISABLE_SHORT_OPEN_TAG takes precedence
+echo json_encode(token_get_all('<?xml', TOKEN_DISABLE_SHORT_OPEN_TAG|TOKEN_ENABLE_SHORT_OPEN_TAG)), "\n";
+echo json_encode(PhpToken::tokenize('<?xml', TOKEN_DISABLE_SHORT_OPEN_TAG)), "\n";
+echo json_encode(PhpToken::tokenize('<?xml', TOKEN_ENABLE_SHORT_OPEN_TAG)), "\n";
+?>
+--EXPECTF--
+[[%d,"<?xml",1]]
+[[%d,"<?",1],[%d,"xml",1]]
+[[%d,"<?xml",1]]
+[{"id":%d,"text":"<?xml","line":1,"pos":0}]
+[{"id":%d,"text":"<?","line":1,"pos":0},{"id":%d,"text":"xml","line":1,"pos":2}]

ext/tokenizer/tokenizer.c

@@ -319,7 +319,7 @@ static void add_token(
 	zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &token);
 }
 
-static bool tokenize(zval *return_value, zend_string *source, zend_class_entry *token_class)
+static bool tokenize(zval *return_value, zend_string *source, zend_class_entry *token_class, bool short_tags)
 {
 	zval source_zval;
 	zend_lex_state original_lex_state;
@@ -335,6 +335,7 @@ static bool tokenize(zval *return_value, zend_string *source, zend_class_entry *
 	zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC());
 
 	LANG_SCNG(yy_state) = yycINITIAL;
+	LANG_SCNG(short_tags) = short_tags;
 	zend_hash_init(&interned_strings, 0, NULL, NULL, 0);
 	array_init(return_value);
 
@@ -452,7 +453,7 @@ void on_event(
 }
 
 static bool tokenize_parse(
-		zval *return_value, zend_string *source, zend_class_entry *token_class)
+		zval *return_value, zend_string *source, zend_class_entry *token_class, bool short_tags)
 {
 	zval source_zval;
 	struct event_context ctx;
@@ -478,6 +479,7 @@ static bool tokenize_parse(
 	LANG_SCNG(yy_state) = yycINITIAL;
 	LANG_SCNG(on_event) = on_event;
 	LANG_SCNG(on_event_context) = &ctx;
+	LANG_SCNG(short_tags) = short_tags;
 
 	if((success = (zendparse() == SUCCESS))) {
 		ZVAL_COPY_VALUE(return_value, &token_stream);
@@ -500,14 +502,23 @@ static bool tokenize_parse(
 static bool tokenize_common(
 		zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class)
 {
+	bool result;
+	bool short_tags;
+	if (flags & (TOKEN_ENABLE_SHORT_OPEN_TAG|TOKEN_DISABLE_SHORT_OPEN_TAG)) {
+		/* TOKEN_DISABLE_SHORT_OPEN_TAG takes precedence over TOKEN_ENABLE_SHORT_OPEN_TAG */
+		short_tags = (flags & TOKEN_DISABLE_SHORT_OPEN_TAG) == 0;
+	} else {
+		short_tags = CG(short_tags);
+	}
 	if (flags & TOKEN_PARSE) {
-		return tokenize_parse(return_value, source, token_class);
+		result = tokenize_parse(return_value, source, token_class, short_tags);
 	} else {
-		int success = tokenize(return_value, source, token_class);
+		int success = tokenize(return_value, source, token_class, short_tags);
 		/* Normal token_get_all() should not throw. */
 		zend_clear_exception();
-		return success;
+		result = success;
 	}
+	return result;
 }
 
 /* }}} */

ext/tokenizer/tokenizer.stub.php

@@ -7,6 +7,16 @@
  * @cvalue TOKEN_PARSE
  */
 const TOKEN_PARSE = UNKNOWN;
+/**
+ * @var int
+ * @cvalue TOKEN_ENABLE_SHORT_OPEN_TAG
+ */
+const TOKEN_ENABLE_SHORT_OPEN_TAG = UNKNOWN;
+/**
+ * @var int
+ * @cvalue TOKEN_DISABLE_SHORT_OPEN_TAG
+ */
+const TOKEN_DISABLE_SHORT_OPEN_TAG = UNKNOWN;
 
 function token_get_all(string $code, int $flags = 0): array {}