TokenIterator - detect newline

Author: ondrejmirtes

src/Parser/TokenIterator.php

@@ -9,6 +9,7 @@
 use function count;
 use function in_array;
 use function strlen;
+use function substr;
 
 class TokenIterator
 {
@@ -25,6 +26,9 @@ class TokenIterator
 	/** @var list<int> */
 	private $skippedTokenTypes = [Lexer::TOKEN_HORIZONTAL_WS];
 
+	/** @var string|null */
+	private $newline = null;
+
 	/**
 	 * @param list<array{string, int, int}> $tokens
 	 */
@@ -144,6 +148,12 @@ public function consumeTokenType(int $tokenType): void
 			$this->throwError($tokenType);
 		}
 
+		if ($tokenType === Lexer::TOKEN_PHPDOC_EOL) {
+			if ($this->newline === null) {
+				$this->detectNewline();
+			}
+		}
+
 		$this->index++;
 		$this->skipIrrelevantTokens();
 	}
@@ -184,13 +194,30 @@ public function tryConsumeTokenType(int $tokenType): bool
 			return false;
 		}
 
+		if ($tokenType === Lexer::TOKEN_PHPDOC_EOL) {
+			if ($this->newline === null) {
+				$this->detectNewline();
+			}
+		}
+
 		$this->index++;
 		$this->skipIrrelevantTokens();
 
 		return true;
 	}
 
 
+	private function detectNewline(): void
+	{
+		$value = $this->currentTokenValue();
+		if (substr($value, 0, 2) === "\r\n") {
+			$this->newline = "\r\n";
+		} elseif (substr($value, 0, 1) === "\n") {
+			$this->newline = "\n";
+		}
+	}
+
+
 	public function getSkippedHorizontalWhiteSpaceIfAny(): string
 	{
 		if ($this->index > 0 && $this->tokens[$this->index - 1][Lexer::TYPE_OFFSET] === Lexer::TOKEN_HORIZONTAL_WS) {
@@ -339,6 +366,11 @@ public function hasTokenImmediatelyAfter(int $pos, int $expectedTokenType): bool
 		return false;
 	}
 
+	public function getDetectedNewline(): ?string
+	{
+		return $this->newline;
+	}
+
 	/**
 	 * Whether the given position is immediately surrounded by parenthesis.
 	 */

tests/PHPStan/Parser/TokenIteratorTest.php

@@ -0,0 +1,58 @@
+<?php declare(strict_types = 1);
+
+namespace PHPStan\PhpDocParser\Parser;
+
+use PHPStan\PhpDocParser\Lexer\Lexer;
+use PHPUnit\Framework\TestCase;
+use const PHP_EOL;
+
+class TokenIteratorTest extends TestCase
+{
+
+	/**
+	 * @return iterable<array{string, ?string}>
+	 */
+	public function dataGetDetectedNewline(): iterable
+	{
+		yield [
+			'/** @param Foo $a */',
+			null,
+		];
+
+		yield [
+			'/**' . "\n" .
+			' * @param Foo $a' . "\n" .
+			' */',
+			"\n",
+		];
+
+		yield [
+			'/**' . "\r\n" .
+			' * @param Foo $a' . "\r\n" .
+			' */',
+			"\r\n",
+		];
+
+		yield [
+			'/**' . PHP_EOL .
+			' * @param Foo $a' . PHP_EOL .
+			' */',
+			PHP_EOL,
+		];
+	}
+
+	/**
+	 * @dataProvider dataGetDetectedNewline
+	 */
+	public function testGetDetectedNewline(string $phpDoc, ?string $expectedNewline): void
+	{
+		$lexer = new Lexer(true);
+		$tokens = new TokenIterator($lexer->tokenize($phpDoc));
+		$constExprParser = new ConstExprParser();
+		$typeParser = new TypeParser($constExprParser);
+		$phpDocParser = new PhpDocParser($typeParser, $constExprParser);
+		$phpDocParser->parse($tokens);
+		$this->assertSame($expectedNewline, $tokens->getDetectedNewline());
+	}
+
+}