Improve string literal

Author: imteekay

baselines/reference/stringLIteral.tree.baseline

@@ -22,7 +22,7 @@
       },
       "init": {
         "kind": "Literal",
-        "value": "'singleQuote'"
+        "value": "singleQuote"
       }
     },
     {
@@ -33,11 +33,15 @@
       },
       "init": {
         "kind": "Literal",
-        "value": "\"doubleQuote\""
+        "value": "double'Quote"
       }
     },
     {
-      "kind": "EmptyStatement"
+      "kind": "ExpressionStatement",
+      "expr": {
+        "kind": "Identifier",
+        "text": "(missing)"
+      }
     }
   ]
 }

src/lex.ts

@@ -1,4 +1,4 @@
-import { Token, Lexer } from './types';
+import { Token, Lexer, CharCodes } from './types';
 
 const keywords = {
   function: Token.Function,
@@ -41,8 +41,7 @@ export function lex(s: string): Lexer {
           ? keywords[text as keyof typeof keywords]
           : Token.Identifier;
     } else if (['"', "'"].includes(s.charAt(pos))) {
-      scanForward((c) => /[_a-zA-Z0-9'"]/.test(c));
-      text = s.slice(start, pos);
+      text = scanString();
       token = Token.String;
     } else {
       pos++;
@@ -66,6 +65,62 @@ export function lex(s: string): Lexer {
   function scanForward(pred: (x: string) => boolean) {
     while (pos < s.length && pred(s.charAt(pos))) pos++;
   }
+
+  function scanString() {
+    const quote = s.charCodeAt(pos);
+    pos++;
+
+    let stringValue = '';
+    let start = pos;
+
+    while (true) {
+      if (pos >= s.length) {
+        // report unterminated string literal error
+      }
+
+      const char = s.charCodeAt(pos);
+
+      if (char === quote) {
+        stringValue += s.slice(start, pos);
+        pos++;
+        break;
+      }
+
+      if (char === CharCodes.backslash) {
+        stringValue += s.slice(start, pos);
+        stringValue += scanEscapeSequence();
+        start = pos;
+        continue;
+      }
+
+      pos++;
+    }
+
+    return stringValue;
+  }
+
+  function scanEscapeSequence() {
+    pos++;
+    const char = s.charCodeAt(pos);
+    pos++;
+
+    switch (char) {
+      case CharCodes.b:
+        return '\b';
+      case CharCodes.t:
+        return '\t';
+      case CharCodes.n:
+        return '\n';
+      case CharCodes.r:
+        return '\r';
+      case CharCodes.singleQuote:
+        return "'";
+      case CharCodes.doubleQuote:
+        return '"';
+      default:
+        return '';
+    }
+  }
 }
 
 export function lexAll(s: string) {

src/types.ts

@@ -94,3 +94,13 @@ export type Module = {
 };
 
 export type Type = { id: string };
+
+export enum CharCodes {
+  b = 98,
+  t = 116,
+  n = 110,
+  r = 114,
+  singleQuote = 39,
+  doubleQuote = 34,
+  backslash = 92,
+}

tests/stringLiteral.ts

@@ -1,2 +1,2 @@
 var singleQuote = 'singleQuote';
-var doubleQuote = "doubleQuote";
+var doubleQuote = 'double\'Quote';