TokenizeTest: add additional test cases for text string handling

This adds extra tests for:
* Double quoted strings with interpolated variables.
* Nowdoc
* Heredocs with and without interpolated variables.

Note: for the nowdoc and heredoc tests, the test input is put in single quotes to prevent "nested here/nowdoc", which could skew the tests.
These test cases also need to have a new line after the `;` as otherwise PHP does not tokenize these correctly (and the highlighter is not concerned with fixing incorrect tokenization of PHP itself).

Author: jrfnl

tests/TokenizeTest.php

@@ -426,6 +426,81 @@ public function dataTextStrings()
                 'expected' => <<<'EOL'
 <token_default><?php</token_default>
 <token_keyword>echo </token_keyword><token_string>"Ahoj světe"</token_string><token_keyword>;</token_keyword>
+EOL
+            ),
+            'Double quoted text string with interpolation [1]' => array(
+                'original' => <<<'EOL'
+<?php
+echo "Ahoj $text and more text";
+EOL
+                ,
+                'expected' => <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo </token_keyword><token_string>"Ahoj </token_string><token_default>$text</token_default><token_string> and more text"</token_string><token_keyword>;</token_keyword>
+EOL
+            ),
+            'Double quoted text string with interpolation [2]' => array(
+                'original' => <<<'EOL'
+<?php
+echo "$text and more text";
+EOL
+                ,
+                'expected' => <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo </token_keyword><token_string>"</token_string><token_default>$text</token_default><token_string> and more text"</token_string><token_keyword>;</token_keyword>
+EOL
+            ),
+            'Double quoted text string with interpolation [3]' => array(
+                'original' => <<<'EOL'
+<?php
+echo "Ahoj {$obj->prop} and more text";
+EOL
+                ,
+                'expected' => <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo </token_keyword><token_string>"Ahoj </token_string><token_keyword>{</token_keyword><token_default>$obj</token_default><token_keyword>-></token_keyword><token_default>prop</token_default><token_keyword>}</token_keyword><token_string> and more text"</token_string><token_keyword>;</token_keyword>
+EOL
+            ),
+            'Nowdoc' => array(
+                'original' => '<?php
+echo <<<\'TXT\'
+Text
+TXT;
+',
+                'expected' => <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo <<<'TXT'</token_keyword>
+<token_string>Text</token_string>
+<token_keyword>TXT;</token_keyword>
+
+EOL
+            ),
+            'Heredoc' => array(
+                'original' => '<?php
+echo <<<TXT
+Text
+TXT;
+',
+                'expected' => <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo <<<TXT</token_keyword>
+<token_string>Text</token_string>
+<token_keyword>TXT;</token_keyword>
+
+EOL
+            ),
+            'Heredoc with interpolation' => array(
+                'original' => '<?php
+echo <<<TXT
+Text $text
+TXT;
+',
+                'expected' => <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo <<<TXT</token_keyword>
+<token_string>Text </token_string><token_default>$text</token_default>
+<token_keyword>TXT;</token_keyword>
+
 EOL
             ),
         );