Skip to content

Commit

Permalink
Tokenizer::replaceTabsInToken(): add tests
Browse files Browse the repository at this point in the history 
Add tests for the logic in the `Tokenizer::replaceTabsInToken()` method.
  • Loading branch information
@jrfnl
jrfnl committed Oct 25, 2024
1 parent 3f42c78 commit c16696c
Show file tree
Hide file tree
Showing 9 changed files with 996 additions and 1 deletion.
2 changes: 1 addition & 1 deletion src/Tokenizers/Tokenizer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ public function replaceTabsInToken(&$token, $prefix=' ', $padding=' ', $tabWidth
if ($content !== '') {
$newContent .= $content;
if ($checkEncoding === true) {
// Not using the default encoding, so take a bit more care.
// Not using ASCII encoding, so take a bit more care.
$oldLevel = error_reporting();
error_reporting(0);
$contentLength = iconv_strlen($content, $this->config->encoding);
Expand Down
107 changes: 107 additions & 0 deletions tests/Core/Tokenizers/Tokenizer/CreatePositionMapTabWidth0Test.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
<?php
/**
* Tests the tab replacement logic.
*
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
* @copyright 2024 PHPCSStandards and contributors
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;

/**
* Tab replacement test using tab width 0, means no tab replacement will take place.
*
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
*/
final class CreatePositionMapTabWidth0Test extends ReplaceTabsInTokenTestCase
{

/**
* The tab width setting to use when tokenizing the file.
*
* @var integer
*/
protected $tabWidth = 0;


/**
* Data provider helper.
*
* @see ReplaceTabsInTokenTestCase::dataTabReplacement()
*
* @return array<string, array<string, int|string|null>>
*/
public static function getTabReplacementExpected()
{
return [
'Tab indentation' => [
'length' => 2,
'content' => ' ',
'orig_content' => null,
],
'Mixed tab/space indentation' => [
'length' => 3,
'content' => ' ',
'orig_content' => null,
],
'Inline: single tab in text string' => [
'length' => 15,
'content' => "'tab separated'",
'orig_content' => null,
],
'Inline: single tab between each word in text string' => [
'length' => 24,
'content' => '"tab $between each word"',
'orig_content' => null,
],
'Inline: multiple tabs in heredoc' => [
'length' => 15,
'content' => 'tab separated
',
'orig_content' => null,
],
'Inline: multiple tabs between each word in nowdoc' => [
'length' => 27,
'content' => 'tab between each word
',
'orig_content' => null,
],
'Inline: mixed spaces/tabs in text string' => [
'length' => 20,
'content' => "'tab separated'",
'orig_content' => null,
],
'Inline: mixed spaces/tabs between each word in text string' => [
'length' => 31,
'content' => '"tab $between each word"',
'orig_content' => null,
],
'Inline: tab becomes single space in comment (with tabwidth 4)' => [
'length' => 50,
'content' => '// -123 With tabwidth 4, the tab size should be 1.
',
'orig_content' => null,
],
'Inline: tab becomes 2 spaces in comment (with tabwidth 4)' => [
'length' => 52,
'content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
'orig_content' => null,
],
'Inline: tab becomes 3 spaces in doc comment string (with tabwidth 4)' => [
'length' => 45,
'content' => '-1 With tabwidth 4, the tab size should be 3.',
'orig_content' => null,
],
'Inline: tab becomes 4 spaces in comment (with tabwidth 4)' => [
'length' => 47,
'content' => '// - With tabwidth 4, the tab size should be 4.
',
'orig_content' => null,
],
];

}//end getTabReplacementExpected()


}//end class
124 changes: 124 additions & 0 deletions tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenMiscTest.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
<?php
/**
* Tests the tab replacement logic.
*
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
* @copyright 2024 PHPCSStandards and contributors
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;

use PHP_CodeSniffer\Files\DummyFile;
use PHP_CodeSniffer\Ruleset;
use PHP_CodeSniffer\Tests\ConfigDouble;
use PHPUnit\Framework\TestCase;

/**
* Miscellaneous tests for tab replacement.
*
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::replaceTabsInToken
*/
final class ReplaceTabsInTokenMiscTest extends TestCase
{


/**
* Test that when no tab width is set or passed, the tab width will be set to 1.
*
* @return void
*/
public function testTabWidthNotSet()
{
$config = new ConfigDouble();
$ruleset = new Ruleset($config);

$content = <<<EOD
<?php
echo 'foo';
EOD;
$phpcsFile = new DummyFile($content, $ruleset, $config);
$phpcsFile->parse();

$tokens = $phpcsFile->getTokens();
$target = $phpcsFile->findNext(T_WHITESPACE, 0);

// Verify initial state.
$this->assertTrue(is_int($target), 'Target token was not found');
$this->assertSame(' ', $tokens[$target]['content'], 'Content after initial parsing does not contain tabs');
$this->assertSame(2, $tokens[$target]['length'], 'Length after initial parsing is not as expected');
$this->assertArrayNotHasKey('orig_content', $tokens[$target], "Key 'orig_content' found in the initial token array.");

$phpcsFile->tokenizer->replaceTabsInToken($tokens[$target]);

// Verify tab replacement.
$this->assertSame(' ', $tokens[$target]['content'], 'Content after tab replacement is not as expected');
$this->assertSame(2, $tokens[$target]['length'], 'Length after tab replacement is not as expected');
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");

}//end testTabWidthNotSet()


/**
* Test that the length calculation handles text in non-ascii encodings correctly.
*
* @requires extension iconv
*
* @return void
*/
public function testLengthSettingRespectsEncoding()
{
$config = new ConfigDouble();
$config->tabWidth = 4;
$ruleset = new Ruleset($config);

$content = <<<EOD
<?php
echo 'пасха пасха';
EOD;
$phpcsFile = new DummyFile($content, $ruleset, $config);
$phpcsFile->parse();

$tokens = $phpcsFile->getTokens();
$target = $phpcsFile->findNext(T_CONSTANT_ENCAPSED_STRING, 0);

$this->assertTrue(is_int($target), 'Target token was not found');
$this->assertSame("'пасха пасха'", $tokens[$target]['content'], 'Content is not as expected');
$this->assertSame(17, $tokens[$target]['length'], 'Length is not as expected');
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");
$this->assertSame("'пасха пасха'", $tokens[$target]['orig_content'], 'Orig_content is not as expected');

}//end testLengthSettingRespectsEncoding()


/**
* Test that the length calculation falls back to byte length if iconv detects an illegal character.
*
* @requires extension iconv
*
* @return void
*/
public function testLengthSettingFallsBackToBytesWhenTextContainsIllegalChars()
{
$config = new ConfigDouble();
$config->tabWidth = 4;
$ruleset = new Ruleset($config);

$content = <<<EOD
<?php
echo "aa\xC3\xC3 \xC3\xB8aa";
EOD;
$phpcsFile = new DummyFile($content, $ruleset, $config);
$phpcsFile->parse();

$tokens = $phpcsFile->getTokens();
$target = $phpcsFile->findNext(T_CONSTANT_ENCAPSED_STRING, 0);

$this->assertTrue(is_int($target), 'Target token was not found');
$this->assertSame(11, $tokens[$target]['length'], 'Length is not as expected');
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");

}//end testLengthSettingFallsBackToBytesWhenTextContainsIllegalChars()


}//end class
111 changes: 111 additions & 0 deletions tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenTabWidth1Test.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
<?php
/**
* Tests the tab replacement logic.
*
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
* @copyright 2024 PHPCSStandards and contributors
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;

/**
* Tab replacement test using tab width 1.
*
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::replaceTabsInToken
*/
final class ReplaceTabsInTokenTabWidth1Test extends ReplaceTabsInTokenTestCase
{

/**
* The tab width setting to use when tokenizing the file.
*
* @var integer
*/
protected $tabWidth = 1;


/**
* Data provider helper.
*
* @see ReplaceTabsInTokenTestCase::dataTabReplacement()
*
* @return array<string, array<string, int|string>>
*/
public static function getTabReplacementExpected()
{
return [
'Tab indentation' => [
'length' => 2,
'content' => ' ',
'orig_content' => ' ',
],
'Mixed tab/space indentation' => [
'length' => 3,
'content' => ' ',
'orig_content' => ' ',
],
'Inline: single tab in text string' => [
'length' => 15,
'content' => "'tab separated'",
'orig_content' => "'tab separated'",
],
'Inline: single tab between each word in text string' => [
'length' => 24,
'content' => '"tab $between each word"',
'orig_content' => '"tab $between each word"',
],
'Inline: multiple tabs in heredoc' => [
'length' => 15,
'content' => 'tab separated
',
'orig_content' => 'tab separated
',
],
'Inline: multiple tabs between each word in nowdoc' => [
'length' => 27,
'content' => 'tab between each word
',
'orig_content' => 'tab between each word
',
],
'Inline: mixed spaces/tabs in text string' => [
'length' => 20,
'content' => "'tab separated'",
'orig_content' => "'tab separated'",
],
'Inline: mixed spaces/tabs between each word in text string' => [
'length' => 31,
'content' => '"tab $between each word"',
'orig_content' => '"tab $between each word"',
],
'Inline: tab becomes single space in comment (with tabwidth 4)' => [
'length' => 50,
'content' => '// -123 With tabwidth 4, the tab size should be 1.
',
'orig_content' => '// -123 With tabwidth 4, the tab size should be 1.
',
],
'Inline: tab becomes 2 spaces in comment (with tabwidth 4)' => [
'length' => 52,
'content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
'orig_content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
],
'Inline: tab becomes 3 spaces in doc comment string (with tabwidth 4)' => [
'length' => 45,
'content' => '-1 With tabwidth 4, the tab size should be 3.',
'orig_content' => '-1 With tabwidth 4, the tab size should be 3.',
],
'Inline: tab becomes 4 spaces in comment (with tabwidth 4)' => [
'length' => 47,
'content' => '// - With tabwidth 4, the tab size should be 4.
',
'orig_content' => '// - With tabwidth 4, the tab size should be 4.
',
],
];

}//end getTabReplacementExpected()


}//end class
Loading

0 comments on commit c16696c

Please sign in to comment.