TypeParser - support comments at EOL with //

Author: shmax

README.md

@@ -83,7 +83,7 @@ use PHPStan\PhpDocParser\Printer\Printer;
 
 // basic setup with enabled required lexer attributes
 
-$config = new ParserConfig(usedAttributes: ['lines' => true, 'indexes' => true]);
+$config = new ParserConfig(usedAttributes: ['lines' => true, 'indexes' => true, 'comments' => true]);
 $lexer = new Lexer($config);
 $constExprParser = new ConstExprParser($config);
 $typeParser = new TypeParser($config, $constExprParser);

src/Ast/Attribute.php

@@ -13,4 +13,6 @@ final class Attribute
 
 	public const ORIGINAL_NODE = 'originalNode';
 
+	public const COMMENTS = 'comments';
+
 }

src/Ast/Comment.php

@@ -0,0 +1,28 @@
+<?php declare(strict_types = 1);
+
+namespace PHPStan\PhpDocParser\Ast;
+
+use function trim;
+
+class Comment
+{
+
+	public string $text;
+
+	public int $startLine;
+
+	public int $startIndex;
+
+	public function __construct(string $text, int $startLine = -1, int $startIndex = -1)
+	{
+		$this->text = $text;
+		$this->startLine = $startLine;
+		$this->startIndex = $startIndex;
+	}
+
+	public function getReformattedText(): string
+	{
+		return trim($this->text);
+	}
+
+}

src/Ast/NodeAttributes.php

@@ -15,6 +15,10 @@ trait NodeAttributes
 	 */
 	public function setAttribute(string $key, $value): void
 	{
+		if ($value === null) {
+			unset($this->attributes[$key]);
+			return;
+		}
 		$this->attributes[$key] = $value;
 	}
 

src/Lexer/Lexer.php

@@ -51,6 +51,8 @@ class Lexer
 	public const TOKEN_NEGATED = 35;
 	public const TOKEN_ARROW = 36;
 
+	public const TOKEN_COMMENT = 37;
+
 	public const TOKEN_LABELS = [
 		self::TOKEN_REFERENCE => '\'&\'',
 		self::TOKEN_UNION => '\'|\'',
@@ -66,6 +68,7 @@ class Lexer
 		self::TOKEN_OPEN_CURLY_BRACKET => '\'{\'',
 		self::TOKEN_CLOSE_CURLY_BRACKET => '\'}\'',
 		self::TOKEN_COMMA => '\',\'',
+		self::TOKEN_COMMENT => '\'//\'',
 		self::TOKEN_COLON => '\':\'',
 		self::TOKEN_VARIADIC => '\'...\'',
 		self::TOKEN_DOUBLE_COLON => '\'::\'',
@@ -160,6 +163,7 @@ private function generateRegexp(): string
 			self::TOKEN_CLOSE_CURLY_BRACKET => '\\}',
 
 			self::TOKEN_COMMA => ',',
+			self::TOKEN_COMMENT => '\/\/[^\\r\\n]*(?=\n|\r|\*/)',
 			self::TOKEN_VARIADIC => '\\.\\.\\.',
 			self::TOKEN_DOUBLE_COLON => '::',
 			self::TOKEN_DOUBLE_ARROW => '=>',

src/Parser/PhpDocParser.php

@@ -116,9 +116,19 @@ public function parse(TokenIterator $tokens): Ast\PhpDoc\PhpDocNode
 
 			$tokens->forwardToTheEnd();
 
+			$comments = $tokens->flushComments();
+			if ($comments !== []) {
+				throw new LogicException('Comments should already be flushed');
+			}
+
 			return $this->enrichWithAttributes($tokens, new Ast\PhpDoc\PhpDocNode([$this->enrichWithAttributes($tokens, $tag, $startLine, $startIndex)]), 1, 0);
 		}
 
+		$comments = $tokens->flushComments();
+		if ($comments !== []) {
+			throw new LogicException('Comments should already be flushed');
+		}
+
 		return $this->enrichWithAttributes($tokens, new Ast\PhpDoc\PhpDocNode($children), 1, 0);
 	}
 

src/Parser/TokenIterator.php

@@ -3,6 +3,7 @@
 namespace PHPStan\PhpDocParser\Parser;
 
 use LogicException;
+use PHPStan\PhpDocParser\Ast\Comment;
 use PHPStan\PhpDocParser\Lexer\Lexer;
 use function array_pop;
 use function assert;
@@ -19,7 +20,10 @@ class TokenIterator
 
 	private int $index;
 
-	/** @var int[] */
+	/** @var list<Comment> */
+	private array $comments = [];
+
+	/** @var list<array{int, list<Comment>}> */
 	private array $savePoints = [];
 
 	/** @var list<int> */
@@ -152,8 +156,7 @@ public function consumeTokenType(int $tokenType): void
 			}
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 	}
 
 
@@ -166,8 +169,7 @@ public function consumeTokenValue(int $tokenType, string $tokenValue): void
 			$this->throwError($tokenType, $tokenValue);
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 	}
 
 
@@ -178,12 +180,20 @@ public function tryConsumeTokenValue(string $tokenValue): bool
 			return false;
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 
 		return true;
 	}
 
+	/**
+	 * @return list<Comment>
+	 */
+	public function flushComments(): array
+	{
+		$res = $this->comments;
+		$this->comments = [];
+		return $res;
+	}
 
 	/** @phpstan-impure */
 	public function tryConsumeTokenType(int $tokenType): bool
@@ -198,14 +208,15 @@ public function tryConsumeTokenType(int $tokenType): bool
 			}
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 
 		return true;
 	}
 
 
-	/** @phpstan-impure */
+	/**
+	 * @deprecated Use skipNewLineTokensAndConsumeComments instead (when parsing a type)
+	 */
 	public function skipNewLineTokens(): void
 	{
 		if (!$this->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
@@ -218,6 +229,29 @@ public function skipNewLineTokens(): void
 	}
 
 
+	public function skipNewLineTokensAndConsumeComments(): void
+	{
+		if ($this->currentTokenType() === Lexer::TOKEN_COMMENT) {
+			$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
+			$this->next();
+		}
+
+		if (!$this->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
+			return;
+		}
+
+		do {
+			$foundNewLine = $this->tryConsumeTokenType(Lexer::TOKEN_PHPDOC_EOL);
+			if ($this->currentTokenType() !== Lexer::TOKEN_COMMENT) {
+				continue;
+			}
+
+			$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
+			$this->next();
+		} while ($foundNewLine === true);
+	}
+
+
 	private function detectNewline(): void
 	{
 		$value = $this->currentTokenValue();
@@ -293,7 +327,7 @@ public function forwardToTheEnd(): void
 
 	public function pushSavePoint(): void
 	{
-		$this->savePoints[] = $this->index;
+		$this->savePoints[] = [$this->index, $this->comments];
 	}
 
 
@@ -305,9 +339,9 @@ public function dropSavePoint(): void
 
 	public function rollback(): void
 	{
-		$index = array_pop($this->savePoints);
-		assert($index !== null);
-		$this->index = $index;
+		$savepoint = array_pop($this->savePoints);
+		assert($savepoint !== null);
+		[$this->index, $this->comments] = $savepoint;
 	}