Skip to content

Support php 8.0's "Treat namespaced names as single token" and "?->" #331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/Parser.php
Original file line number Diff line number Diff line change
Expand Up @@ -1199,7 +1199,7 @@ private function parseTemplateStringExpression($parentNode) {
$token = $this->getCurrentToken();
if ($token->kind === TokenKind::OpenBracketToken) {
return $this->parseTemplateStringSubscriptExpression($var);
} else if ($token->kind === TokenKind::ArrowToken) {
} else if ($token->kind === TokenKind::ArrowToken || $token->kind === TokenKind::QuestionArrowToken) {
return $this->parseTemplateStringMemberAccessExpression($var);
} else {
return $var;
Expand Down Expand Up @@ -1248,7 +1248,7 @@ private function parseTemplateStringMemberAccessExpression($expression) : Member
$expression->parent = $memberAccessExpression;

$memberAccessExpression->dereferencableExpression = $expression;
$memberAccessExpression->arrowToken = $this->eat1(TokenKind::ArrowToken);
$memberAccessExpression->arrowToken = $this->eat(TokenKind::ArrowToken, TokenKind::QuestionArrowToken);
$memberAccessExpression->memberName = $this->eat1(TokenKind::Name);

return $memberAccessExpression;
Expand Down Expand Up @@ -2711,7 +2711,7 @@ private function parsePostfixExpressionRest($expression, $allowUpdateExpression
return $expression;
}

if ($tokenKind === TokenKind::ArrowToken) {
if ($tokenKind === TokenKind::ArrowToken || $tokenKind === TokenKind::QuestionArrowToken) {
$expression = $this->parseMemberAccessExpression($expression);
return $this->parsePostfixExpressionRest($expression);
}
Expand Down Expand Up @@ -2836,7 +2836,7 @@ private function parseMemberAccessExpression($expression):MemberAccessExpression
$expression->parent = $memberAccessExpression;

$memberAccessExpression->dereferencableExpression = $expression;
$memberAccessExpression->arrowToken = $this->eat1(TokenKind::ArrowToken);
$memberAccessExpression->arrowToken = $this->eat(TokenKind::ArrowToken, TokenKind::QuestionArrowToken);
$memberAccessExpression->memberName = $this->parseMemberName($memberAccessExpression);

return $memberAccessExpression;
Expand Down
55 changes: 54 additions & 1 deletion src/PhpTokenizer.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@
// The replacement value is arbitrary - it just has to be different from other values of token constants.
define(__NAMESPACE__ . '\T_COALESCE_EQUAL', defined('T_COALESCE_EQUAL') ? constant('T_COALESCE_EQUAL') : 'T_COALESCE_EQUAL');
define(__NAMESPACE__ . '\T_FN', defined('T_FN') ? constant('T_FN') : 'T_FN');
// If this predaates PHP 8.0, T_MATCH is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
// If this predates PHP 8.0, T_MATCH is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
define(__NAMESPACE__ . '\T_MATCH', defined('T_MATCH') ? constant('T_MATCH') : 'T_MATCH');
define(__NAMESPACE__ . '\T_NULLSAFE_OBJECT_OPERATOR', defined('T_NULLSAFE_OBJECT_OPERATOR') ? constant('T_NULLSAFE_OBJECT_OPERATOR') : 'T_MATCH');

/**
* Tokenizes content using PHP's built-in `token_get_all`, and converts to "lightweight" Token representation.
Expand Down Expand Up @@ -130,6 +131,57 @@ public static function getTokensArrayFromContent(
$arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
$start = $fullStart = $pos;
break;
case \PHP_VERSION_ID >= 80000 ? \T_NAME_QUALIFIED : -1000:
case \PHP_VERSION_ID >= 80000 ? \T_NAME_FULLY_QUALIFIED : -1001:
// NOTE: This switch is called on every token of every file being parsed, so this traded performance for readability.
//
// PHP's Opcache is able to optimize switches that are exclusively known longs,
// but not switches that mix strings and longs or have unknown longs.
// Longs are only known if they're declared within the same *class* or an internal constant (tokenizer).
//
// For some reason, the SWITCH_LONG opcode was not generated when the expression was part of a class constant.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the explanation!

// (seen with php -d opcache.opt_debug_level=0x20000)
//
// Use negative values because that's not expected to overlap with token kinds that token_get_all() will return.
//
// T_NAME_* was added in php 8.0 to forbid whitespace between parts of names.
// Here, emulate the tokenization of php 7 by splitting it up into 1 or more tokens.
foreach (\explode('\\', $token[1]) as $i => $name) {
if ($i) {
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1 + $start - $fullStart);
$start++;
$fullStart = $start;
}
if ($name === '') {
continue;
}
// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
$len = \strlen($name);
$arr[] = new Token(TokenKind::Name, $fullStart, $start, $len + $start - $fullStart);
$start += $len;
$fullStart = $start;
}
break;
case \PHP_VERSION_ID >= 80000 ? \T_NAME_RELATIVE : -1002:
// This is a namespace-relative name: namespace\...
foreach (\explode('\\', $token[1]) as $i => $name) {
$len = \strlen($name);
if (!$i) {
$arr[] = new Token(TokenKind::NamespaceKeyword, $fullStart, $start, $len + $start - $fullStart);
$start += $len;
$fullStart = $start;
continue;
}
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1);
$start++;

// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
$arr[] = new Token(TokenKind::Name, $start, $start, $len);

$start += $len;
$fullStart = $start;
}
break;
case \T_COMMENT:
case \T_DOC_COMMENT:
if ($treatCommentsAsTrivia) {
Expand Down Expand Up @@ -256,6 +308,7 @@ protected static function tokenGetAll(string $content, $parseContext): array
"}" => TokenKind::CloseBraceToken,
"." => TokenKind::DotToken,
T_OBJECT_OPERATOR => TokenKind::ArrowToken,
T_NULLSAFE_OBJECT_OPERATOR => TokenKind::QuestionArrowToken,
T_INC => TokenKind::PlusPlusToken,
T_DEC => TokenKind::MinusMinusToken,
T_POW => TokenKind::AsteriskAsteriskToken,
Expand Down
1 change: 1 addition & 0 deletions src/TokenKind.php
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ class TokenKind {
const BacktickToken = 260;
const QuestionToken = 261;
const QuestionQuestionEqualsToken = 262;
const QuestionArrowToken = 263;

const DecimalLiteralToken = 301;
const OctalLiteralToken = 302;
Expand Down
1 change: 1 addition & 0 deletions src/TokenStringMaps.php
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ class TokenStringMaps {
"^=" => TokenKind::CaretEqualsToken,
"|=" => TokenKind::BarEqualsToken,
"," => TokenKind::CommaToken,
"?->" => TokenKind::QuestionArrowToken,
"??" => TokenKind::QuestionQuestionToken,
"??=" => TokenKind::QuestionQuestionEqualsToken,
"<=>" => TokenKind::LessThanEqualsGreaterThanToken,
Expand Down
2 changes: 2 additions & 0 deletions tests/cases/parser80/nullsafe_operator1.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
<?php
$x = $a?->b;
1 change: 1 addition & 0 deletions tests/cases/parser80/nullsafe_operator1.php.diag
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
67 changes: 67 additions & 0 deletions tests/cases/parser80/nullsafe_operator1.php.tree
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{
"SourceFileNode": {
"statementList": [
{
"InlineHtml": {
"scriptSectionEndTag": null,
"text": null,
"scriptSectionStartTag": {
"kind": "ScriptSectionStartTag",
"textLength": 6
}
}
},
{
"ExpressionStatement": {
"expression": {
"AssignmentExpression": {
"leftOperand": {
"Variable": {
"dollar": null,
"name": {
"kind": "VariableName",
"textLength": 2
}
}
},
"operator": {
"kind": "EqualsToken",
"textLength": 1
},
"byRef": null,
"rightOperand": {
"MemberAccessExpression": {
"dereferencableExpression": {
"Variable": {
"dollar": null,
"name": {
"kind": "VariableName",
"textLength": 2
}
}
},
"arrowToken": {
"kind": "QuestionArrowToken",
"textLength": 3
},
"memberName": {
"kind": "Name",
"textLength": 1
}
}
}
}
},
"semicolon": {
"kind": "SemicolonToken",
"textLength": 1
}
}
}
],
"endOfFileToken": {
"kind": "EndOfFileToken",
"textLength": 0
}
}
}
2 changes: 2 additions & 0 deletions tests/cases/parser80/nullsafe_operator2.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
<?php
$a?->foo(1);
1 change: 1 addition & 0 deletions tests/cases/parser80/nullsafe_operator2.php.diag
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
81 changes: 81 additions & 0 deletions tests/cases/parser80/nullsafe_operator2.php.tree
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
{
"SourceFileNode": {
"statementList": [
{
"InlineHtml": {
"scriptSectionEndTag": null,
"text": null,
"scriptSectionStartTag": {
"kind": "ScriptSectionStartTag",
"textLength": 6
}
}
},
{
"ExpressionStatement": {
"expression": {
"CallExpression": {
"callableExpression": {
"MemberAccessExpression": {
"dereferencableExpression": {
"Variable": {
"dollar": null,
"name": {
"kind": "VariableName",
"textLength": 2
}
}
},
"arrowToken": {
"kind": "QuestionArrowToken",
"textLength": 3
},
"memberName": {
"kind": "Name",
"textLength": 3
}
}
},
"openParen": {
"kind": "OpenParenToken",
"textLength": 1
},
"argumentExpressionList": {
"ArgumentExpressionList": {
"children": [
{
"ArgumentExpression": {
"byRefToken": null,
"dotDotDotToken": null,
"expression": {
"NumericLiteral": {
"children": {
"kind": "IntegerLiteralToken",
"textLength": 1
}
}
}
}
}
]
}
},
"closeParen": {
"kind": "CloseParenToken",
"textLength": 1
}
}
},
"semicolon": {
"kind": "SemicolonToken",
"textLength": 1
}
}
}
],
"endOfFileToken": {
"kind": "EndOfFileToken",
"textLength": 0
}
}
}
1 change: 1 addition & 0 deletions tests/cases/parser80/nullsafe_operator3.php.diag
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
69 changes: 69 additions & 0 deletions tests/cases/parser80/nullsafe_operator3.php.tree
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"SourceFileNode": {
"statementList": [
{
"InlineHtml": {
"scriptSectionEndTag": null,
"text": null,
"scriptSectionStartTag": {
"kind": "ScriptSectionStartTag",
"textLength": 6
}
}
},
{
"ExpressionStatement": {
"expression": {
"ObjectCreationExpression": {
"newKeword": {
"kind": "NewKeyword",
"textLength": 3
},
"classTypeDesignator": {
"MemberAccessExpression": {
"dereferencableExpression": {
"Variable": {
"dollar": null,
"name": {
"kind": "VariableName",
"textLength": 2
}
}
},
"arrowToken": {
"kind": "QuestionArrowToken",
"textLength": 3
},
"memberName": {
"kind": "Name",
"textLength": 1
}
}
},
"openParen": {
"kind": "OpenParenToken",
"textLength": 1
},
"argumentExpressionList": null,
"closeParen": {
"kind": "CloseParenToken",
"textLength": 1
},
"classBaseClause": null,
"classInterfaceClause": null,
"classMembers": null
}
},
"semicolon": {
"kind": "SemicolonToken",
"textLength": 1
}
}
}
],
"endOfFileToken": {
"kind": "EndOfFileToken",
"textLength": 0
}
}
}