From d96c7d6489476230bc19acea19159eaef056eede Mon Sep 17 00:00:00 2001 From: jrfnl Date: Wed, 15 May 2024 17:57:40 +0200 Subject: [PATCH 1/4] Tokenizers/Comment: add tests The `Tokenizers\Comment` class did not have any tests associated with it. This commit fixes that and documents the existing behaviour. Note: code coverage is as high as it can go, but not 100%. The reason for this, is the tokenizer debug statements, which are conditional on a verbosity flag, which is turned off for the tests. Loosely related to 484. --- .../Tokenizer/Comment/CommentTestCase.php | 117 ++++ .../Tokenizer/Comment/LiveCoding1Test.inc | 6 + .../Tokenizer/Comment/LiveCoding1Test.php | 69 ++ .../Tokenizer/Comment/LiveCoding2Test.inc | 5 + .../Tokenizer/Comment/LiveCoding2Test.php | 68 ++ .../Tokenizer/Comment/LiveCoding3Test.inc | 4 + .../Tokenizer/Comment/LiveCoding3Test.php | 62 ++ .../Tokenizer/Comment/LiveCoding4Test.inc | 7 + .../Tokenizer/Comment/LiveCoding4Test.php | 76 +++ .../Comment/MultiLineDocBlockTest.inc | 81 +++ .../Comment/MultiLineDocBlockTest.php | 439 ++++++++++++ .../PhpcsAnnotationsInDocBlockTest.inc | 116 ++++ .../PhpcsAnnotationsInDocBlockTest.php | 637 ++++++++++++++++++ .../Comment/SingleLineDocBlockTest.inc | 20 + .../Comment/SingleLineDocBlockTest.php | 173 +++++ 15 files changed, 1880 insertions(+) create mode 100644 tests/Core/Tokenizer/Comment/CommentTestCase.php create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding1Test.inc create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding1Test.php create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding2Test.inc create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding2Test.php create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding3Test.inc create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding3Test.php create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding4Test.inc create mode 100644 tests/Core/Tokenizer/Comment/LiveCoding4Test.php create mode 100644 tests/Core/Tokenizer/Comment/MultiLineDocBlockTest.inc create mode 100644 tests/Core/Tokenizer/Comment/MultiLineDocBlockTest.php create mode 100644 tests/Core/Tokenizer/Comment/PhpcsAnnotationsInDocBlockTest.inc create mode 100644 tests/Core/Tokenizer/Comment/PhpcsAnnotationsInDocBlockTest.php create mode 100644 tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc create mode 100644 tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.php diff --git a/tests/Core/Tokenizer/Comment/CommentTestCase.php b/tests/Core/Tokenizer/Comment/CommentTestCase.php new file mode 100644 index 0000000000..0dbd977156 --- /dev/null +++ b/tests/Core/Tokenizer/Comment/CommentTestCase.php @@ -0,0 +1,117 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase; +use PHP_CodeSniffer\Util\Tokens; + +/** + * Base class for testing DocBlock comment tokenization. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +abstract class CommentTestCase extends AbstractTokenizerTestCase +{ + + + /** + * Test whether the docblock opener and closer have the expected extra keys. + * + * @param string $marker The comment prefacing the target token. + * @param int $closerOffset The offset of the closer from the opener. + * @param array $expectedTags The expected tags offsets array. + * + * @dataProvider dataDocblockOpenerCloser + * + * @return void + */ + public function testDocblockOpenerCloser($marker, $closerOffset, $expectedTags) + { + $tokens = $this->phpcsFile->getTokens(); + $target = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]); + + $opener = $tokens[$target]; + + $this->assertArrayHasKey('comment_closer', $opener, 'Comment opener: comment_closer index is not set'); + $this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set'); + + $expectedCloser = ($target + $closerOffset); + $this->assertSame($expectedCloser, $opener['comment_closer'], 'Comment opener: comment_closer not set to the expected stack pointer'); + + // Update the tags expectations. + foreach ($expectedTags as $i => $ptr) { + $expectedTags[$i] += $target; + } + + $this->assertSame($expectedTags, $opener['comment_tags'], 'Comment opener: recorded tags do not match expected tags'); + + $closer = $tokens[$opener['comment_closer']]; + + $this->assertArrayHasKey('comment_opener', $closer, 'Comment closer: comment_opener index is not set'); + $this->assertSame($target, $closer['comment_opener'], 'Comment closer: comment_opener not set to the expected stack pointer'); + + }//end testDocblockOpenerCloser() + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + abstract public static function dataDocblockOpenerCloser(); + + + /** + * Test helper. Check a token sequence complies with an expected token sequence. + * + * @param int $startPtr The position in the file to start checking from. + * @param array> $expectedSequence The consecutive token constants and their contents to expect. + * + * @return void + */ + protected function checkTokenSequence($startPtr, array $expectedSequence) + { + $tokens = $this->phpcsFile->getTokens(); + + $sequenceKey = 0; + $sequenceCount = count($expectedSequence); + + for ($i = $startPtr; $sequenceKey < $sequenceCount; $i++, $sequenceKey++) { + $currentItem = $expectedSequence[$sequenceKey]; + $expectedCode = key($currentItem); + $expectedType = Tokens::tokenName($expectedCode); + $expectedContent = current($currentItem); + $errorMsgSuffix = PHP_EOL.'(StackPtr: '.$i.' | Position in sequence: '.$sequenceKey.' | Expected: '.$expectedType.')'; + + $this->assertSame( + $expectedCode, + $tokens[$i]['code'], + 'Token tokenized as '.Tokens::tokenName($tokens[$i]['code']).', not '.$expectedType.' (code)'.$errorMsgSuffix + ); + + $this->assertSame( + $expectedType, + $tokens[$i]['type'], + 'Token tokenized as '.$tokens[$i]['type'].', not '.$expectedType.' (type)'.$errorMsgSuffix + ); + + $this->assertSame( + $expectedContent, + $tokens[$i]['content'], + 'Token content did not match expectations'.$errorMsgSuffix + ); + }//end for + + }//end checkTokenSequence() + + +}//end class diff --git a/tests/Core/Tokenizer/Comment/LiveCoding1Test.inc b/tests/Core/Tokenizer/Comment/LiveCoding1Test.inc new file mode 100644 index 0000000000..a43c7d9b26 --- /dev/null +++ b/tests/Core/Tokenizer/Comment/LiveCoding1Test.inc @@ -0,0 +1,6 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +/** + * Tests that unclosed docblocks during live coding are handled correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +final class LiveCoding1Test extends CommentTestCase +{ + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + public static function dataDocblockOpenerCloser() + { + return [ + 'live coding: unclosed docblock, no blank line at end of file' => [ + 'marker' => '/* testLiveCoding */', + 'closerOffset' => 8, + 'expectedTags' => [], + ], + ]; + + }//end dataDocblockOpenerCloser() + + + /** + * Verify tokenization of the DocBlock. + * + * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. + * + * @return void + */ + public function testLiveCoding() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Unclosed docblock, live coding.... with no blank line at end of file.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testLiveCoding() + + +}//end class diff --git a/tests/Core/Tokenizer/Comment/LiveCoding2Test.inc b/tests/Core/Tokenizer/Comment/LiveCoding2Test.inc new file mode 100644 index 0000000000..b113645b53 --- /dev/null +++ b/tests/Core/Tokenizer/Comment/LiveCoding2Test.inc @@ -0,0 +1,5 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +/** + * Tests that unclosed docblocks during live coding are handled correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +final class LiveCoding2Test extends CommentTestCase +{ + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + public static function dataDocblockOpenerCloser() + { + return [ + 'live coding: unclosed docblock with blank line at end of file' => [ + 'marker' => '/* testLiveCoding */', + 'closerOffset' => 7, + 'expectedTags' => [], + ], + ]; + + }//end dataDocblockOpenerCloser() + + + /** + * Verify tokenization of the DocBlock. + * + * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. + * + * @return void + */ + public function testLiveCoding() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Unclosed docblock, live coding.... with a blank line at end of file.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_CLOSE_TAG => ''], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testLiveCoding() + + +}//end class diff --git a/tests/Core/Tokenizer/Comment/LiveCoding3Test.inc b/tests/Core/Tokenizer/Comment/LiveCoding3Test.inc new file mode 100644 index 0000000000..9b81a434d9 --- /dev/null +++ b/tests/Core/Tokenizer/Comment/LiveCoding3Test.inc @@ -0,0 +1,4 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +/** + * Tests that unclosed docblocks during live coding are handled correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +final class LiveCoding3Test extends CommentTestCase +{ + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + public static function dataDocblockOpenerCloser() + { + return [ + 'live coding: unclosed docblock, no contents, no blank line at end of file' => [ + 'marker' => '/* testLiveCoding */', + 'closerOffset' => 1, + 'expectedTags' => [], + ], + ]; + + }//end dataDocblockOpenerCloser() + + + /** + * Verify tokenization of the DocBlock. + * + * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. + * + * @return void + */ + public function testLiveCoding() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_CLOSE_TAG => ''], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testLiveCoding() + + +}//end class diff --git a/tests/Core/Tokenizer/Comment/LiveCoding4Test.inc b/tests/Core/Tokenizer/Comment/LiveCoding4Test.inc new file mode 100644 index 0000000000..1561e7156c --- /dev/null +++ b/tests/Core/Tokenizer/Comment/LiveCoding4Test.inc @@ -0,0 +1,7 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +/** + * Tests that unclosed docblocks during live coding are handled correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +final class LiveCoding4Test extends CommentTestCase +{ + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + public static function dataDocblockOpenerCloser() + { + return [ + 'live coding: unclosed docblock, trailing whitespace on last line, no blank line at end of file' => [ + 'marker' => '/* testLiveCoding */', + 'closerOffset' => 15, + 'expectedTags' => [], + ], + ]; + + }//end dataDocblockOpenerCloser() + + + /** + * Verify tokenization of the DocBlock. + * + * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. + * + * @return void + */ + public function testLiveCoding() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'The last line of this test must have trailing whitespace.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'So, be careful when saving this file!'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => ''], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testLiveCoding() + + +}//end class diff --git a/tests/Core/Tokenizer/Comment/MultiLineDocBlockTest.inc b/tests/Core/Tokenizer/Comment/MultiLineDocBlockTest.inc new file mode 100644 index 0000000000..f33afcd6ae --- /dev/null +++ b/tests/Core/Tokenizer/Comment/MultiLineDocBlockTest.inc @@ -0,0 +1,81 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +/** + * Tests that multiline docblocks are tokenized correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +final class MultiLineDocBlockTest extends CommentTestCase +{ + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + public static function dataDocblockOpenerCloser() + { + return [ + 'Multi line docblock: no contents' => [ + 'marker' => '/* testEmptyDocblock */', + 'closerOffset' => 3, + 'expectedTags' => [], + ], + 'Multi line docblock: variety of text and tags' => [ + 'marker' => '/* testMultilineDocblock */', + 'closerOffset' => 95, + // phpcs:ignore Squiz.Arrays.ArrayDeclaration.SingleLineNotAllowed + 'expectedTags' => [21, 29, 36, 46, 56, 63, 73, 80, 90], + ], + 'Multi line docblock: no leading stars' => [ + 'marker' => '/* testMultilineDocblockNoStars */', + 'closerOffset' => 32, + // phpcs:ignore Squiz.Arrays.ArrayDeclaration.SingleLineNotAllowed + 'expectedTags' => [10, 16, 21, 27], + ], + 'Multi line docblock: indented' => [ + 'marker' => '/* testMultilineDocblockIndented */', + 'closerOffset' => 60, + // phpcs:ignore Squiz.Arrays.ArrayDeclaration.SingleLineNotAllowed + 'expectedTags' => [21, 28, 38, 45, 55], + ], + 'Multi line docblock: opener not on own line' => [ + 'marker' => '/* testMultilineDocblockOpenerNotOnOwnLine */', + 'closerOffset' => 10, + 'expectedTags' => [], + ], + 'Multi line docblock: closer not on own line' => [ + 'marker' => '/* testMultilineDocblockCloserNotOnOwnLine */', + 'closerOffset' => 11, + 'expectedTags' => [], + ], + 'Multi line docblock: stars not aligned' => [ + 'marker' => '/* testMultilineDocblockStarsNotAligned */', + 'closerOffset' => 26, + 'expectedTags' => [], + ], + ]; + + }//end dataDocblockOpenerCloser() + + + /** + * Verify tokenization of an empty, multi-line DocBlock. + * + * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. + * + * @return void + */ + public function testEmptyDocblock() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testEmptyDocblock() + + + /** + * Verify tokenization of a multi-line DocBlock containing all possible tokens. + * + * @return void + */ + public function testMultilineDocblock() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'This is a multi-line docblock.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'With blank lines, stars, tags, and tag descriptions.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tagWithoutDescription'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@since'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => '10.3'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@deprecated'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => '11.5'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@requires'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'PHP 7.1 -- PHPUnit tag.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tag-with-dashes-is-suppported'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Description.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tag_with_underscores'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Description.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@param'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'string $p1 Description 1.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@param'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'int|false $p2 Description 2.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@return'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'void'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultilineDocblock() + + + /** + * Verify tokenization of a multi-line DocBlock with extra starts for the opener/closer and no stars on the lines between. + * + * @return void + */ + public function testMultilineDocblockNoStars() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/****'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'This is a multi-line docblock, but the lines are not marked with stars.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Then again, the opener and closer have an abundance of stars.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@since'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => '10.3'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@param'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'string $p1 Description 1.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@param'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'int|false $p2 Description 2.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@return'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'void'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '**/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultilineDocblockNoStars() + + + /** + * Verify tokenization of a multi-line, indented DocBlock. + * + * @return void + */ + public function testMultilineDocblockIndented() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'This is a multi-line indented docblock.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'With blank lines, stars, tags, and tag descriptions.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@since'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => '10.3'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@deprecated'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => '11.5'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@param'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'string $p1 Description 1.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@param'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'int|false $p2 Description 2.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@return'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'void'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultilineDocblockIndented() + + + /** + * Verify tokenization of a multi-line DocBlock, where the opener is not on its own line. + * + * @return void + */ + public function testMultilineDocblockOpenerNotOnOwnLine() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Start of description'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'description continued.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultilineDocblockOpenerNotOnOwnLine() + + + /** + * Verify tokenization of a multi-line DocBlock, where the closer is not on its own line. + * + * @return void + */ + public function testMultilineDocblockCloserNotOnOwnLine() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Start of description'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'description continued. '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultilineDocblockCloserNotOnOwnLine() + + + /** + * Verify tokenization of a multi-line DocBlock with inconsistent indentation. + * + * @return void + */ + public function testMultilineDocblockStarsNotAligned() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Start of description.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Line below this is missing a star.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Text'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Star indented.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Closer indented.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultilineDocblockStarsNotAligned() + + +}//end class diff --git a/tests/Core/Tokenizer/Comment/PhpcsAnnotationsInDocBlockTest.inc b/tests/Core/Tokenizer/Comment/PhpcsAnnotationsInDocBlockTest.inc new file mode 100644 index 0000000000..d74f68d246 --- /dev/null +++ b/tests/Core/Tokenizer/Comment/PhpcsAnnotationsInDocBlockTest.inc @@ -0,0 +1,116 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +/** + * Tests that PHPCS native annotations in docblocks are tokenized correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +final class PhpcsAnnotationsInDocBlockTest extends CommentTestCase +{ + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + public static function dataDocblockOpenerCloser() + { + return [ + 'Single-line: @phpcs:ignoreFile annotation' => [ + 'marker' => '/* testSingleLineDocIgnoreFileAnnotation */', + 'closerOffset' => 3, + 'expectedTags' => [], + ], + 'Single-line: @phpcs:ignore annotation' => [ + 'marker' => '/* testSingleLineDocIgnoreAnnotation */', + 'closerOffset' => 3, + 'expectedTags' => [], + ], + 'Single-line: @phpcs:disable annotation' => [ + 'marker' => '/* testSingleLineDocDisableAnnotation */', + 'closerOffset' => 3, + 'expectedTags' => [], + ], + 'Single-line: @phpcs:enable annotation; no whitespace' => [ + 'marker' => '/* testSingleLineDocEnableAnnotationNoWhitespace */', + 'closerOffset' => 2, + 'expectedTags' => [], + ], + + 'Multi-line: @phpcs:ignoreFile at the start' => [ + 'marker' => '/* testMultiLineDocIgnoreFileAnnotationAtStart */', + 'closerOffset' => 13, + 'expectedTags' => [], + ], + 'Multi-line: @phpcs:ignore at the start' => [ + 'marker' => '/* testMultiLineDocIgnoreAnnotationAtStart */', + 'closerOffset' => 13, + 'expectedTags' => [10], + ], + 'Multi-line: @phpcs:disable at the start' => [ + 'marker' => '/* testMultiLineDocDisableAnnotationAtStart */', + 'closerOffset' => 13, + 'expectedTags' => [], + ], + 'Multi-line: @phpcs:enable at the start' => [ + 'marker' => '/* testMultiLineDocEnableAnnotationAtStart */', + 'closerOffset' => 18, + 'expectedTags' => [13], + ], + + 'Multi-line: @phpcs:ignoreFile in the middle' => [ + 'marker' => '/* testMultiLineDocIgnoreFileAnnotationInMiddle */', + 'closerOffset' => 21, + 'expectedTags' => [], + ], + 'Multi-line: @phpcs:ignore in the middle' => [ + 'marker' => '/* testMultiLineDocIgnoreAnnotationInMiddle */', + 'closerOffset' => 23, + 'expectedTags' => [5], + ], + 'Multi-line: @phpcs:disable in the middle' => [ + 'marker' => '/* testMultiLineDocDisableAnnotationInMiddle */', + 'closerOffset' => 26, + 'expectedTags' => [21], + ], + 'Multi-line: @phpcs:enable in the middle' => [ + 'marker' => '/* testMultiLineDocEnableAnnotationInMiddle */', + 'closerOffset' => 24, + 'expectedTags' => [21], + ], + + 'Multi-line: @phpcs:ignoreFile at the end' => [ + 'marker' => '/* testMultiLineDocIgnoreFileAnnotationAtEnd */', + 'closerOffset' => 16, + 'expectedTags' => [5], + ], + 'Multi-line: @phpcs:ignore at the end' => [ + 'marker' => '/* testMultiLineDocIgnoreAnnotationAtEnd */', + 'closerOffset' => 16, + 'expectedTags' => [], + ], + 'Multi-line: @phpcs:disable at the end' => [ + 'marker' => '/* testMultiLineDocDisableAnnotationAtEnd */', + 'closerOffset' => 18, + 'expectedTags' => [5], + ], + 'Multi-line: @phpcs:enable at the end' => [ + 'marker' => '/* testMultiLineDocEnableAnnotationAtEnd */', + 'closerOffset' => 16, + 'expectedTags' => [], + ], + ]; + + }//end dataDocblockOpenerCloser() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignoreFile annotation. + * + * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. + * + * @return void + */ + public function testSingleLineDocIgnoreFileAnnotation() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE_FILE => '@phpcs:ignoreFile '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocIgnoreFileAnnotation() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignore annotation. + * + * @return void + */ + public function testSingleLineDocIgnoreAnnotation() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE => '@phpcs:ignore Stnd.Cat.SniffName -- With reason '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocIgnoreAnnotation() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS disable annotation. + * + * @return void + */ + public function testSingleLineDocDisableAnnotation() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_DISABLE => '@phpcs:disable Stnd.Cat.SniffName,Stnd.Other '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocDisableAnnotation() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS enable annotation. + * + * @return void + */ + public function testSingleLineDocEnableAnnotationNoWhitespace() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_PHPCS_ENABLE => '@phpcs:enable Stnd.Cat.SniffName'], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocEnableAnnotationNoWhitespace() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignoreFile annotation at the start. + * + * @return void + */ + public function testMultiLineDocIgnoreFileAnnotationAtStart() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE_FILE => '@phpcs:ignoreFile'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Something.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocIgnoreFileAnnotationAtStart() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignore annotation at the start. + * + * @return void + */ + public function testMultiLineDocIgnoreAnnotationAtStart() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE => '@phpcs:ignore Stnd.Cat.SniffName'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tag'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocIgnoreAnnotationAtStart() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS disable annotation at the start. + * + * @return void + */ + public function testMultiLineDocDisableAnnotationAtStart() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_DISABLE => '@phpcs:disable Stnd.Cat.SniffName -- Ensure PHPCS annotations are also retokenized correctly.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Something.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocDisableAnnotationAtStart() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS enable annotation at the start. + * + * @return void + */ + public function testMultiLineDocEnableAnnotationAtStart() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_ENABLE => '@phpcs:enable Stnd.Cat,Stnd.Other'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tag'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'With description.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocEnableAnnotationAtStart() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignoreFile annotation in the middle. + * + * @return void + */ + public function testMultiLineDocIgnoreFileAnnotationInMiddle() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Check tokenization of PHPCS annotations within docblocks.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE_FILE => '@phpcs:ignoreFile'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Something.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocIgnoreFileAnnotationInMiddle() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignore annotation in the middle. + * + * @return void + */ + public function testMultiLineDocIgnoreAnnotationInMiddle() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tagBefore'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'With Description'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE => '@phpcs:ignore Stnd.Cat.SniffName'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Something.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocIgnoreAnnotationInMiddle() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS disable annotation in the middle. + * + * @return void + */ + public function testMultiLineDocDisableAnnotationInMiddle() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Check tokenization of PHPCS annotations within docblocks.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_DISABLE => '@phpcs:disable Stnd.Cat.SniffName -- Ensure PHPCS annotations are also retokenized correctly.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tagAfter'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'With Description'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocDisableAnnotationInMiddle() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS enable annotation in the middle. + * + * @return void + */ + public function testMultiLineDocEnableAnnotationInMiddle() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Check tokenization of PHPCS annotations within docblocks.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_ENABLE => '@phpcs:enable Stnd.Cat,Stnd.Other'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tagAfter'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocEnableAnnotationInMiddle() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignoreFile annotation at the end. + * + * @return void + */ + public function testMultiLineDocIgnoreFileAnnotationAtEnd() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tagBefore'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE_FILE => '@phpcs:ignoreFile'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocIgnoreFileAnnotationAtEnd() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS ignore annotation at the end. + * + * @return void + */ + public function testMultiLineDocIgnoreAnnotationAtEnd() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Check tokenization of PHPCS annotations within docblocks.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_IGNORE => '@phpcs:ignore Stnd.Cat.SniffName'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocIgnoreAnnotationAtEnd() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS disable annotation at the end. + * + * @return void + */ + public function testMultiLineDocDisableAnnotationAtEnd() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@tagBefore'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'With Description.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_DISABLE => '@phpcs:disable Stnd.Cat.SniffName -- Ensure PHPCS annotations are also retokenized correctly.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocDisableAnnotationAtEnd() + + + /** + * Verify tokenization of a single line DocBlock containing a PHPCS enable annotation at the end. + * + * @return void + */ + public function testMultiLineDocEnableAnnotationAtEnd() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Check tokenization of PHPCS annotations within docblocks.'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STAR => '*'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_PHPCS_ENABLE => '@phpcs:enable Stnd.Cat,Stnd.Other'], + [T_DOC_COMMENT_WHITESPACE => "\n"], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testMultiLineDocEnableAnnotationAtEnd() + + +}//end class diff --git a/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc new file mode 100644 index 0000000000..5775ea563e --- /dev/null +++ b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc @@ -0,0 +1,20 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment; + +/** + * Tests that single line docblocks are tokenized correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\Comment + */ +final class SingleLineDocBlockTest extends CommentTestCase +{ + + + /** + * Data provider. + * + * @see testDocblockOpenerCloser() + * + * @return array>> + */ + public static function dataDocblockOpenerCloser() + { + return [ + 'Single line docblock: only whitespace' => [ + 'marker' => '/* testEmptyDocblockWithWhiteSpace */', + 'closerOffset' => 2, + 'expectedTags' => [], + ], + 'Single line docblock: just text' => [ + 'marker' => '/* testSingleLineDocblockNoTag */', + 'closerOffset' => 3, + 'expectedTags' => [], + ], + 'Single line docblock: @var type before name' => [ + 'marker' => '/* testSingleLineDocblockWithTag1 */', + 'closerOffset' => 5, + 'expectedTags' => [2], + ], + 'Single line docblock: @var name before type' => [ + 'marker' => '/* testSingleLineDocblockWithTag2 */', + 'closerOffset' => 5, + 'expectedTags' => [2], + ], + 'Single line docblock: @see with description' => [ + 'marker' => '/* testSingleLineDocblockWithTag3 */', + 'closerOffset' => 5, + 'expectedTags' => [2], + ], + ]; + + }//end dataDocblockOpenerCloser() + + + /** + * Verify tokenization of an empty, single line DocBlock. + * + * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. + * + * @return void + */ + public function testEmptyDocblockWithWhiteSpace() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testEmptyDocblockWithWhiteSpace() + + + /** + * Verify tokenization of a single line DocBlock. + * + * @return void + */ + public function testSingleLineDocblockNoTag() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Just some text '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocblockNoTag() + + + /** + * Verify tokenization of a single line DocBlock with a tag. + * + * @return void + */ + public function testSingleLineDocblockWithTag1() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@var'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => '\SomeClass[] $var '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocblockWithTag1() + + + /** + * Verify tokenization of a single line DocBlock with a tag. + * + * @return void + */ + public function testSingleLineDocblockWithTag2() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@var'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => '$var \SomeClass[] '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocblockWithTag2() + + + /** + * Verify tokenization of a single line DocBlock with a tag. + * + * @return void + */ + public function testSingleLineDocblockWithTag3() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_TAG => '@see'], + [T_DOC_COMMENT_WHITESPACE => ' '], + [T_DOC_COMMENT_STRING => 'Something::Else '], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testSingleLineDocblockWithTag3() + + +}//end class From c54cb1043eac4c8f9252a6f4c75b27663955a1fe Mon Sep 17 00:00:00 2001 From: jrfnl Date: Wed, 15 May 2024 19:59:11 +0200 Subject: [PATCH 2/4] Tokenizers/PHP: bug fix - empty block comment This commit fixes an edge case tokenizer bug, where a - completely empty, not even whitespace - _block comment_, would be tokenized as a docblock. Without this commit, the `/**/` code snippet was tokenized as: ``` 8 | L07 | C 1 | CC 0 | ( 0) | T_DOC_COMMENT_OPEN_TAG | [ 4]: /**/ 9 | L07 | C 5 | CC 0 | ( 0) | T_DOC_COMMENT_CLOSE_TAG | [ 0]: ``` With the fix applied, it will be tokenized as: ``` 8 | L07 | C 1 | CC 0 | ( 0) | T_COMMENT | [ 4]: /**/ ``` --- src/Tokenizers/PHP.php | 2 +- .../Comment/SingleLineDocBlockTest.inc | 3 +++ .../Comment/SingleLineDocBlockTest.php | 20 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 55baf41689..6e500e6cd1 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -786,7 +786,7 @@ protected function tokenize($string) if ($tokenIsArray === true && ($token[0] === T_DOC_COMMENT - || ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0)) + || ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0 && $token[1] !== '/**/')) ) { $commentTokens = $commentTokenizer->tokenizeString($token[1], $this->eolChar, $newStackPtr); foreach ($commentTokens as $commentToken) { diff --git a/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc index 5775ea563e..923e0fc1b2 100644 --- a/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc +++ b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc @@ -1,5 +1,8 @@ '/**/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', [T_COMMENT, T_DOC_COMMENT_OPEN_TAG]); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testEmptyBlockCommentNoWhiteSpace() + + /** * Verify tokenization of an empty, single line DocBlock. * From 252675cc908032aba9fd17e446c28180412339f4 Mon Sep 17 00:00:00 2001 From: jrfnl Date: Wed, 15 May 2024 18:00:02 +0200 Subject: [PATCH 3/4] Tokenizers/Comment: bug fix - empty docblock This commit fixes an edge case tokenizer bug, where a - completely empty, not even whitespace - _DocBlock_, would not be tokenized correctly. Without this commit, the `/***/` code snippet was tokenized as: ``` 13 | L10 | C 1 | CC 0 | ( 0) | T_DOC_COMMENT_OPEN_TAG | [ 5]: /***/ 14 | L10 | C 6 | CC 0 | ( 0) | T_DOC_COMMENT_CLOSE_TAG | [ 0]: ``` With the fix applied, it will be tokenized as: ``` 13 | L10 | C 1 | CC 0 | ( 0) | T_DOC_COMMENT_OPEN_TAG | [ 3]: /** 14 | L10 | C 4 | CC 0 | ( 0) | T_DOC_COMMENT_CLOSE_TAG | [ 2]: */ ``` --- src/Tokenizers/Comment.php | 13 +++++++--- .../Comment/SingleLineDocBlockTest.inc | 3 +++ .../Comment/SingleLineDocBlockTest.php | 26 ++++++++++++++++++- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/Tokenizers/Comment.php b/src/Tokenizers/Comment.php index 335e296cb9..523ce9e41a 100644 --- a/src/Tokenizers/Comment.php +++ b/src/Tokenizers/Comment.php @@ -41,9 +41,16 @@ public function tokenizeString($string, $eolChar, $stackPtr) extra star when they are used for function and class comments. */ - $char = ($numChars - strlen(ltrim($string, '/*'))); - $openTag = substr($string, 0, $char); - $string = ltrim($string, '/*'); + $char = ($numChars - strlen(ltrim($string, '/*'))); + $lastChars = substr($string, -2); + if ($char === $numChars && $lastChars === '*/') { + // Edge case: docblock without whitespace or contents. + $openTag = substr($string, 0, -2); + $string = $lastChars; + } else { + $openTag = substr($string, 0, $char); + $string = ltrim($string, '/*'); + } $tokens[$stackPtr] = [ 'content' => $openTag, diff --git a/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc index 923e0fc1b2..88b05ea43c 100644 --- a/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc +++ b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.inc @@ -3,6 +3,9 @@ /* testEmptyBlockCommentNoWhiteSpace */ /**/ +/* testEmptyDocblockNoWhiteSpace */ +/***/ + /* testEmptyDocblockWithWhiteSpace */ /** */ diff --git a/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.php b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.php index 14c59c607a..e90b573d23 100644 --- a/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.php +++ b/tests/Core/Tokenizer/Comment/SingleLineDocBlockTest.php @@ -28,6 +28,11 @@ final class SingleLineDocBlockTest extends CommentTestCase public static function dataDocblockOpenerCloser() { return [ + 'Single line docblock: empty, no whitespace' => [ + 'marker' => '/* testEmptyDocblockNoWhiteSpace */', + 'closerOffset' => 1, + 'expectedTags' => [], + ], 'Single line docblock: only whitespace' => [ 'marker' => '/* testEmptyDocblockWithWhiteSpace */', 'closerOffset' => 2, @@ -79,12 +84,31 @@ public function testEmptyBlockCommentNoWhiteSpace() /** - * Verify tokenization of an empty, single line DocBlock. + * Verify tokenization of an empty, single line DocBlock without whitespace between the opener and closer. * * @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment. * * @return void */ + public function testEmptyDocblockNoWhiteSpace() + { + $expectedSequence = [ + [T_DOC_COMMENT_OPEN_TAG => '/**'], + [T_DOC_COMMENT_CLOSE_TAG => '*/'], + ]; + + $target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG); + + $this->checkTokenSequence($target, $expectedSequence); + + }//end testEmptyDocblockNoWhiteSpace() + + + /** + * Verify tokenization of an empty, single line DocBlock. + * + * @return void + */ public function testEmptyDocblockWithWhiteSpace() { $expectedSequence = [ From bef6fff85cdab300eecb8d1b458496550fd8a7e1 Mon Sep 17 00:00:00 2001 From: jrfnl Date: Wed, 15 May 2024 20:28:38 +0200 Subject: [PATCH 4/4] Tokenizers/Comment: minor tweaks Girlscouting. * Remove an unnecessary interim variable. * Add a comment clarifying certain code. * Specify the array format in the `@return` tags. --- src/Tokenizers/Comment.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Tokenizers/Comment.php b/src/Tokenizers/Comment.php index 523ce9e41a..b7c6e3745e 100644 --- a/src/Tokenizers/Comment.php +++ b/src/Tokenizers/Comment.php @@ -25,7 +25,7 @@ class Comment * @param string $eolChar The EOL character to use for splitting strings. * @param int $stackPtr The position of the first token in the file. * - * @return array + * @return array>> */ public function tokenizeString($string, $eolChar, $stackPtr) { @@ -81,6 +81,7 @@ public function tokenizeString($string, $eolChar, $stackPtr) ]; if ($closeTag['content'] === false) { + // In PHP < 8.0 substr() can return `false` instead of always returning a string. $closeTag['content'] = ''; } @@ -178,7 +179,7 @@ public function tokenizeString($string, $eolChar, $stackPtr) * @param int $start The position in the string to start processing. * @param int $end The position in the string to end processing. * - * @return array + * @return array> */ private function processLine($string, $eolChar, $start, $end) { @@ -253,7 +254,7 @@ private function processLine($string, $eolChar, $start, $end) * @param int $start The position in the string to start processing. * @param int $end The position in the string to end processing. * - * @return array|null + * @return array|null */ private function collectWhitespace($string, $start, $end) { @@ -270,14 +271,12 @@ private function collectWhitespace($string, $start, $end) return null; } - $token = [ + return [ 'content' => $space, 'code' => T_DOC_COMMENT_WHITESPACE, 'type' => 'T_DOC_COMMENT_WHITESPACE', ]; - return $token; - }//end collectWhitespace()