Skip to content

Commit

Permalink
PHP 8.0: "undo" namespaced names as single token
Browse files Browse the repository at this point in the history
As per the proposal in 3041.

This effectively "undoes" the new PHP 8.0 tokenization of identifier names for PHPCS 3.x.

Includes extensive unit tests to ensure the correct re-tokenization as well as that the rest of the tokenization is not adversely affected by this change.

Includes preventing `function ...` within a group use statement from breaking the retokenization.

Includes fixing the nullable tokenization when combined with any of the new PHP 8 identifier name tokens.
  • Loading branch information
jrfnl committed Aug 30, 2020
1 parent d33a6a9 commit d5c858a
Show file tree
Hide file tree
Showing 5 changed files with 1,531 additions and 1 deletion.
6 changes: 6 additions & 0 deletions package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
</dir>
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
<file baseinstalldir="" name="AllTests.php" role="test" />
Expand Down Expand Up @@ -1979,6 +1981,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
</filelist>
Expand Down Expand Up @@ -2038,6 +2042,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
<ignore name="bin/phpcs.bat" />
Expand Down
87 changes: 86 additions & 1 deletion src/Tokenizers/PHP.php
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,81 @@ protected function tokenize($string)
continue;
}//end if

/*
As of PHP 8.0 fully qualified, partially qualified and namespace relative
identifier names are tokenized differently.
This "undoes" the new tokenization so the tokenization will be the same in
in PHP 5, 7 and 8.
*/

if (PHP_VERSION_ID >= 80000
&& $tokenIsArray === true
&& ($token[0] === T_NAME_QUALIFIED
|| $token[0] === T_NAME_FULLY_QUALIFIED
|| $token[0] === T_NAME_RELATIVE)
) {
$name = $token[1];

if ($token[0] === T_NAME_FULLY_QUALIFIED) {
$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$name = ltrim($name, '\\');
}

if ($token[0] === T_NAME_RELATIVE) {
$newToken = [];
$newToken['code'] = T_NAMESPACE;
$newToken['type'] = 'T_NAMESPACE';
$newToken['content'] = substr($name, 0, 9);
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$name = substr($name, 10);
}

$parts = explode('\\', $name);
$partCount = count($parts);
$lastPart = ($partCount - 1);

foreach ($parts as $i => $part) {
$newToken = [];
$newToken['code'] = T_STRING;
$newToken['type'] = 'T_STRING';
$newToken['content'] = $part;
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

if ($i !== $lastPart) {
$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;
}
}

if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = Util\Tokens::tokenName($token[0]);
$content = Util\Common::prepareForOutput($token[1]);
echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
}

continue;
}//end if

/*
Before PHP 7.0, the "yield from" was tokenized as
T_YIELD, T_WHITESPACE and T_STRING. So look for
Expand Down Expand Up @@ -1122,6 +1197,9 @@ protected function tokenize($string)
}

if ($tokenType === T_STRING
|| $tokenType === T_NAME_FULLY_QUALIFIED
|| $tokenType === T_NAME_RELATIVE
|| $tokenType === T_NAME_QUALIFIED
|| $tokenType === T_ARRAY
|| $tokenType === T_NS_SEPARATOR
) {
Expand Down Expand Up @@ -1278,6 +1356,10 @@ protected function tokenize($string)
tokenized as T_STRING even if it appears to be a different token,
such as when writing code like: function default(): foo
so go forward and change the token type before it is processed.
Note: this should not be done for `function Level\Name` within a
group use statement for the PHP 8 identifier name tokens as it
would interfere with the re-tokenization of those.
*/

if ($tokenIsArray === true
Expand All @@ -1295,7 +1377,10 @@ protected function tokenize($string)
}
}

if ($x < $numTokens && is_array($tokens[$x]) === true) {
if ($x < $numTokens
&& is_array($tokens[$x]) === true
&& $tokens[$x][0] !== T_NAME_QUALIFIED
) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$oldType = Util\Tokens::tokenName($tokens[$x][0]);
echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;
Expand Down
13 changes: 13 additions & 0 deletions src/Util/Tokens.php
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,19 @@
define('T_FN', 'PHPCS_T_FN');
}

// Some PHP 8.0 tokens, replicated for lower versions.
if (defined('T_NAME_QUALIFIED') === false) {
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
}

if (defined('T_NAME_FULLY_QUALIFIED') === false) {
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
}

if (defined('T_NAME_RELATIVE') === false) {
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
}

// Tokens used for parsing doc blocks.
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
Expand Down
147 changes: 147 additions & 0 deletions tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
<?php

/* testNamespaceDeclaration */
namespace Package;

/* testNamespaceDeclarationWithLevels */
namespace Vendor\SubLevel\Domain;

/* testUseStatement */
use ClassName;

/* testUseStatementWithLevels */
use Vendor\Level\Domain;

/* testFunctionUseStatement */
use function function_name;

/* testFunctionUseStatementWithLevels */
use function Vendor\Level\function_in_ns;

/* testConstantUseStatement */
use const CONSTANT_NAME;

/* testConstantUseStatementWithLevels */
use const Vendor\Level\OTHER_CONSTANT;

/* testMultiUseUnqualified */
use UnqualifiedClassName,
/* testMultiUsePartiallyQualified */
Sublevel\PartiallyClassName;

/* testGroupUseStatement */
use Vendor\Level\{
AnotherDomain,
function function_grouped,
const CONSTANT_GROUPED,
Sub\YetAnotherDomain,
function SubLevelA\function_grouped_too,
const SubLevelB\CONSTANT_GROUPED_TOO,
};

/* testClassName */
class MyClass
/* testExtendedFQN */
extends \Vendor\Level\FQN
/* testImplementsRelative */
implements namespace\Name,
/* testImplementsFQN */
\Fully\Qualified,
/* testImplementsUnqualified */
Unqualified,
/* testImplementsPartiallyQualified */
Sub\Level\Name
{
/* testFunctionName */
public function function_name(
/* testTypeDeclarationRelative */
namespace\Name $paramA,

/* testTypeDeclarationFQN */
\Fully\Qualified\Name $paramB,

/* testTypeDeclarationUnqualified */
Unqualified $paramC,

/* testTypeDeclarationPartiallyQualified */
Sublevel\Name $paramD,

/* testReturnTypeFQN */
) : \Name {

try {
/* testFunctionCallRelative */
echo NameSpace\function_name();

/* testFunctionCallFQN */
echo \Vendor\Package\function_name();

/* testFunctionCallUnqualified */
echo function_name();

/* testFunctionPartiallyQualified */
echo Level\function_name();

/* testCatchRelative */
} catch (namespace\SubLevel\Exception $e) {

/* testCatchFQN */
} catch (\Exception $e) {

/* testCatchUnqualified */
} catch (Exception $e) {

/* testCatchPartiallyQualified */
} catch (Level\Exception $e) {
}

/* testNewRelative */
$obj = new namespace\ClassName();

/* testNewFQN */
$obj = new \Vendor\ClassName();

/* testNewUnqualified */
$obj = new ClassName;

/* testNewPartiallyQualified */
$obj = new Level\ClassName;

/* testDoubleColonRelative */
$value = namespace\ClassName::property;

/* testDoubleColonFQN */
$value = \ClassName::static_function();

/* testDoubleColonUnqualified */
$value = ClassName::CONSTANT_NAME;

/* testDoubleColonPartiallyQualified */
$value = Level\ClassName::CONSTANT_NAME['key'];

/* testInstanceOfRelative */
$is = $obj instanceof namespace\ClassName;

/* testInstanceOfFQN */
if ($obj instanceof \Full\ClassName) {}

/* testInstanceOfUnqualified */
if ($a === $b && $obj instanceof ClassName && true) {}

/* testInstanceOfPartiallyQualified */
$is = $obj instanceof Partially\ClassName;
}
}

/* testInvalidInPHP8Whitespace */
namespace \ Sublevel
\ function_name();

/* testInvalidInPHP8Comments */
$value = \Fully
// phpcs:ignore Stnd.Cat.Sniff -- for reasons
\Qualified
/* comment */
\Name
// comment
:: function_name();
Loading

0 comments on commit d5c858a

Please sign in to comment.