Skip to content

Commit e5c46d3

Browse files
jrfnljrfnl
authored
Merge pull request #584 from PHPCSStandards/feature/tokenizer-php-add-tests-heredoc-nowdoc
Tokenizer/PHP: add tests for heredoc/nowdoc tokenization
2 parents a3d11a9 + 3e93f2e commit e5c46d3

File tree

4 files changed

+304
-0
lines changed

4 files changed

+304
-0
lines changed

‎tests/Core/Tokenizer/PHP/HeredocNowdocTest.inc

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?php
2+
3+
/* testHeredocSingleLine */
4+
echo <<<EOD
5+
Some $var text
6+
EOD;
7+
8+
/* testNowdocSingleLine */
9+
echo <<<'MARKER'
10+
Some text
11+
MARKER;
12+
13+
/* testHeredocMultiLine */
14+
echo <<<"😬"
15+
Lorum ipsum
16+
Some $var text
17+
dolor sit amet
18+
😬;
19+
20+
/* testNowdocMultiLine */
21+
echo <<<'multi_line'
22+
Lorum ipsum
23+
Some text
24+
dolor sit amet
25+
multi_line;
26+
27+
/* testHeredocEndsOnBlankLine */
28+
echo <<<EOD
29+
Lorum ipsum
30+
dolor sit amet
31+
32+
EOD;
33+
34+
/* testNowdocEndsOnBlankLine */
35+
echo <<<'EOD'
36+
Lorum ipsum
37+
dolor sit amet
38+
39+
EOD;

‎tests/Core/Tokenizer/PHP/HeredocNowdocTest.php

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
<?php
2+
/**
3+
* Tests the tokenization for heredoc/nowdoc constructs.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
13+
use PHP_CodeSniffer\Util\Tokens;
14+
15+
/**
16+
* Tests the tokenization for heredoc/nowdoc constructs.
17+
*
18+
* Verifies that:
19+
* - Nowdoc opener/closers are retokenized from `T_[START_|END_]HEREDOC` to `T_[START_|END_]NOWDOC`.
20+
* - The contents of the heredoc/nowdoc is tokenized as `T_HEREDOC`/`T_NOWDOC`.
21+
* - Each line of the contents has its own token, which includes the new line char.
22+
*
23+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
24+
*/
25+
final class HeredocNowdocTest extends AbstractTokenizerTestCase
26+
{
27+
28+
29+
/**
30+
* Verify tokenization a heredoc construct.
31+
*
32+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
33+
*
34+
* @return void
35+
*/
36+
public function testHeredocSingleLine()
37+
{
38+
$expectedSequence = [
39+
[T_START_HEREDOC => '<<<EOD'."\n"],
40+
[T_HEREDOC => 'Some $var text'."\n"],
41+
[T_END_HEREDOC => 'EOD'],
42+
];
43+
44+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_START_HEREDOC);
45+
46+
$this->checkTokenSequence($target, $expectedSequence);
47+
48+
}//end testHeredocSingleLine()
49+
50+
51+
/**
52+
* Verify tokenization a nowdoc construct.
53+
*
54+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
55+
*
56+
* @return void
57+
*/
58+
public function testNowdocSingleLine()
59+
{
60+
$expectedSequence = [
61+
[T_START_NOWDOC => "<<<'MARKER'\n"],
62+
[T_NOWDOC => 'Some text'."\n"],
63+
[T_END_NOWDOC => 'MARKER'],
64+
];
65+
66+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_START_NOWDOC);
67+
68+
$this->checkTokenSequence($target, $expectedSequence);
69+
70+
}//end testNowdocSingleLine()
71+
72+
73+
/**
74+
* Verify tokenization a multiline heredoc construct.
75+
*
76+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
77+
*
78+
* @return void
79+
*/
80+
public function testHeredocMultiLine()
81+
{
82+
$expectedSequence = [
83+
[T_START_HEREDOC => '<<<"😬"'."\n"],
84+
[T_HEREDOC => 'Lorum ipsum'."\n"],
85+
[T_HEREDOC => 'Some $var text'."\n"],
86+
[T_HEREDOC => 'dolor sit amet'."\n"],
87+
[T_END_HEREDOC => '😬'],
88+
];
89+
90+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_START_HEREDOC);
91+
92+
$this->checkTokenSequence($target, $expectedSequence);
93+
94+
}//end testHeredocMultiLine()
95+
96+
97+
/**
98+
* Verify tokenization a multiline testNowdocSingleLine construct.
99+
*
100+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
101+
*
102+
* @return void
103+
*/
104+
public function testNowdocMultiLine()
105+
{
106+
$expectedSequence = [
107+
[T_START_NOWDOC => "<<<'multi_line'\n"],
108+
[T_NOWDOC => 'Lorum ipsum'."\n"],
109+
[T_NOWDOC => 'Some text'."\n"],
110+
[T_NOWDOC => 'dolor sit amet'."\n"],
111+
[T_END_NOWDOC => 'multi_line'],
112+
];
113+
114+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_START_NOWDOC);
115+
116+
$this->checkTokenSequence($target, $expectedSequence);
117+
118+
}//end testNowdocMultiLine()
119+
120+
121+
/**
122+
* Verify tokenization a multiline heredoc construct.
123+
*
124+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
125+
*
126+
* @return void
127+
*/
128+
public function testHeredocEndsOnBlankLine()
129+
{
130+
$expectedSequence = [
131+
[T_START_HEREDOC => '<<<EOD'."\n"],
132+
[T_HEREDOC => 'Lorum ipsum'."\n"],
133+
[T_HEREDOC => 'dolor sit amet'."\n"],
134+
[T_HEREDOC => "\n"],
135+
[T_END_HEREDOC => 'EOD'],
136+
];
137+
138+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_START_HEREDOC);
139+
140+
$this->checkTokenSequence($target, $expectedSequence);
141+
142+
}//end testHeredocEndsOnBlankLine()
143+
144+
145+
/**
146+
* Verify tokenization a multiline testNowdocSingleLine construct.
147+
*
148+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
149+
*
150+
* @return void
151+
*/
152+
public function testNowdocEndsOnBlankLine()
153+
{
154+
$expectedSequence = [
155+
[T_START_NOWDOC => "<<<'EOD'\n"],
156+
[T_NOWDOC => 'Lorum ipsum'."\n"],
157+
[T_NOWDOC => 'dolor sit amet'."\n"],
158+
[T_NOWDOC => "\n"],
159+
[T_END_NOWDOC => 'EOD'],
160+
];
161+
162+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_START_NOWDOC);
163+
164+
$this->checkTokenSequence($target, $expectedSequence);
165+
166+
}//end testNowdocEndsOnBlankLine()
167+
168+
169+
/**
170+
* Test helper. Check a token sequence complies with an expected token sequence.
171+
*
172+
* @param int $startPtr The position in the file to start checking from.
173+
* @param array<array<int|string, string>> $expectedSequence The consecutive token constants and their contents to expect.
174+
*
175+
* @return void
176+
*/
177+
private function checkTokenSequence($startPtr, array $expectedSequence)
178+
{
179+
$tokens = $this->phpcsFile->getTokens();
180+
181+
$sequenceKey = 0;
182+
$sequenceCount = count($expectedSequence);
183+
184+
for ($i = $startPtr; $sequenceKey < $sequenceCount; $i++, $sequenceKey++) {
185+
$currentItem = $expectedSequence[$sequenceKey];
186+
$expectedCode = key($currentItem);
187+
$expectedType = Tokens::tokenName($expectedCode);
188+
$expectedContent = current($currentItem);
189+
$errorMsgSuffix = PHP_EOL.'(StackPtr: '.$i.' | Position in sequence: '.$sequenceKey.' | Expected: '.$expectedType.')';
190+
191+
$this->assertSame(
192+
$expectedCode,
193+
$tokens[$i]['code'],
194+
'Token tokenized as '.Tokens::tokenName($tokens[$i]['code']).', not '.$expectedType.' (code)'.$errorMsgSuffix
195+
);
196+
197+
$this->assertSame(
198+
$expectedType,
199+
$tokens[$i]['type'],
200+
'Token tokenized as '.$tokens[$i]['type'].', not '.$expectedType.' (type)'.$errorMsgSuffix
201+
);
202+
203+
$this->assertSame(
204+
$expectedContent,
205+
$tokens[$i]['content'],
206+
'Token content did not match expectations'.$errorMsgSuffix
207+
);
208+
}//end for
209+
210+
}//end checkTokenSequence()
211+
212+
213+
}//end class

‎tests/Core/Tokenizer/PHP/HeredocParseErrorTest.inc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<?php
2+
3+
// This is an intentional parse error. This test should be the only test in the file!
4+
// NOTE: this is NOT a _real_ merge conflict, but a valid test.
5+
6+
/* testUnclosedHeredoc */
7+
<<<<<<< HEAD
8+
$a = 10;
9+
=======
10+
$a = 20;
11+
>>>>>>> master

‎tests/Core/Tokenizer/PHP/HeredocParseErrorTest.php

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<?php
2+
/**
3+
* Tests the tokenization for an unclosed heredoc construct.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
13+
14+
/**
15+
* Tests the tokenization for an unclosed heredoc construct.
16+
*
17+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
18+
*/
19+
final class HeredocParseErrorTest extends AbstractTokenizerTestCase
20+
{
21+
22+
23+
/**
24+
* Verify that a heredoc (and nowdoc) start token is retokenized to T_STRING if no closer is found.
25+
*
26+
* @return void
27+
*/
28+
public function testMergeConflict()
29+
{
30+
$tokens = $this->phpcsFile->getTokens();
31+
32+
$token = $this->getTargetToken('/* testUnclosedHeredoc */', [T_START_HEREDOC, T_STRING], '<<< HEAD'."\n");
33+
$tokenArray = $tokens[$token];
34+
35+
$this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_START_HEREDOC (code)');
36+
$this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_START_HEREDOC (type)');
37+
38+
}//end testMergeConflict()
39+
40+
41+
}//end class

0 commit comments

Comments
 (0)