Skip to content

Commit f547ee3

Browse files
imteekayimteekay
authored
Merge pull request #4 from imteekay/string-literals
 `StringLiteral`
2 parents 444d11e + 459e037 commit f547ee3

14 files changed

+321
-4
lines changed

‎README.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ npm run mtsc ./tests/singleVar.ts
3737
- [ ] Make semicolon a statement ender, not statement separator.
3838
- Hint: You'll need a predicate to peek at the next token and decide if it's the start of an element.
3939
- Bonus: Switch from semicolon to newline as statement ender.
40-
- [ ] Add string literals.
40+
- [x] Add string literals.
4141
- [ ] Add let.
4242
- Then add use-before-declaration errors in the checker.
4343
- Finally, add an ES2015 -> ES5 transform that transforms `let` to `var`.
@@ -50,3 +50,6 @@ npm run mtsc ./tests/singleVar.ts
5050
- [ ] Add an ES5 transformer that converts let -> var.
5151
- [ ] Add function declarations and function calls.
5252
- [ ] Add arrow functions with an appropriate transform in ES5.
53+
- [ ] Add support for the lexer to report errors
54+
- report unterminated string literal error
55+
- [ ] Refactor: rename `Literal` to `NumericLiteral`

‎baselines/reference/singleTypedVar.errors.baseline

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,9 @@
22
{
33
"pos": 17,
44
"message": "Cannot assign initialiser of type 'number' to variable with declared type 'string'."
5+
},
6+
{
7+
"pos": 41,
8+
"message": "Cannot assign initialiser of type 'string' to variable with declared type 'number'."
59
}
610
]

‎baselines/reference/singleTypedVar.js.baseline

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
"var s = 1"
1+
"var s = 1;\nvar n = 'test'"

‎baselines/reference/singleTypedVar.tree.baseline

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@
55
"kind": "Var",
66
"pos": 3
77
}
8+
],
9+
"n": [
10+
{
11+
"kind": "Var",
12+
"pos": 22
13+
}
814
]
915
},
1016
"statements": [
@@ -23,6 +29,22 @@
2329
"value": 1
2430
}
2531
},
32+
{
33+
"kind": "Var",
34+
"name": {
35+
"kind": "Identifier",
36+
"text": "n"
37+
},
38+
"typename": {
39+
"kind": "Identifier",
40+
"text": "number"
41+
},
42+
"init": {
43+
"kind": "StringLiteral",
44+
"value": "test",
45+
"isSingleQuote": true
46+
}
47+
},
2648
{
2749
"kind": "EmptyStatement"
2850
}

‎baselines/reference/stringLiteral.errors.baseline

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎baselines/reference/stringLiteral.js.baseline

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"var singleQuote = 'singleQuote';\nvar doubleQuote = \"doubleQuote\";\nvar escapedSingleQuote = 'escapedSingle\\'Quote';\nvar escapedDoubleQuote = \"escapedDouble\\\"Quote\";\nvar escapedB = 'escaped\\nB';\nvar escapedT = 'escaped\\nT';\nvar escapedN = 'escaped\\nN';\nvar escapedR = 'escaped\\nR'"

‎baselines/reference/stringLiteral.tree.baseline

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
{
2+
"locals": {
3+
"singleQuote": [
4+
{
5+
"kind": "Var",
6+
"pos": 3
7+
}
8+
],
9+
"doubleQuote": [
10+
{
11+
"kind": "Var",
12+
"pos": 36
13+
}
14+
],
15+
"escapedSingleQuote": [
16+
{
17+
"kind": "Var",
18+
"pos": 69
19+
}
20+
],
21+
"escapedDoubleQuote": [
22+
{
23+
"kind": "Var",
24+
"pos": 118
25+
}
26+
],
27+
"escapedB": [
28+
{
29+
"kind": "Var",
30+
"pos": 167
31+
}
32+
],
33+
"escapedT": [
34+
{
35+
"kind": "Var",
36+
"pos": 196
37+
}
38+
],
39+
"escapedN": [
40+
{
41+
"kind": "Var",
42+
"pos": 225
43+
}
44+
],
45+
"escapedR": [
46+
{
47+
"kind": "Var",
48+
"pos": 254
49+
}
50+
]
51+
},
52+
"statements": [
53+
{
54+
"kind": "Var",
55+
"name": {
56+
"kind": "Identifier",
57+
"text": "singleQuote"
58+
},
59+
"init": {
60+
"kind": "StringLiteral",
61+
"value": "singleQuote",
62+
"isSingleQuote": true
63+
}
64+
},
65+
{
66+
"kind": "Var",
67+
"name": {
68+
"kind": "Identifier",
69+
"text": "doubleQuote"
70+
},
71+
"init": {
72+
"kind": "StringLiteral",
73+
"value": "doubleQuote",
74+
"isSingleQuote": false
75+
}
76+
},
77+
{
78+
"kind": "Var",
79+
"name": {
80+
"kind": "Identifier",
81+
"text": "escapedSingleQuote"
82+
},
83+
"init": {
84+
"kind": "StringLiteral",
85+
"value": "escapedSingle'Quote",
86+
"isSingleQuote": true
87+
}
88+
},
89+
{
90+
"kind": "Var",
91+
"name": {
92+
"kind": "Identifier",
93+
"text": "escapedDoubleQuote"
94+
},
95+
"init": {
96+
"kind": "StringLiteral",
97+
"value": "escapedDouble\"Quote",
98+
"isSingleQuote": false
99+
}
100+
},
101+
{
102+
"kind": "Var",
103+
"name": {
104+
"kind": "Identifier",
105+
"text": "escapedB"
106+
},
107+
"init": {
108+
"kind": "StringLiteral",
109+
"value": "escaped\nB",
110+
"isSingleQuote": true
111+
}
112+
},
113+
{
114+
"kind": "Var",
115+
"name": {
116+
"kind": "Identifier",
117+
"text": "escapedT"
118+
},
119+
"init": {
120+
"kind": "StringLiteral",
121+
"value": "escaped\nT",
122+
"isSingleQuote": true
123+
}
124+
},
125+
{
126+
"kind": "Var",
127+
"name": {
128+
"kind": "Identifier",
129+
"text": "escapedN"
130+
},
131+
"init": {
132+
"kind": "StringLiteral",
133+
"value": "escaped\nN",
134+
"isSingleQuote": true
135+
}
136+
},
137+
{
138+
"kind": "Var",
139+
"name": {
140+
"kind": "Identifier",
141+
"text": "escapedR"
142+
},
143+
"init": {
144+
"kind": "StringLiteral",
145+
"value": "escaped\nR",
146+
"isSingleQuote": true
147+
}
148+
},
149+
{
150+
"kind": "EmptyStatement"
151+
}
152+
]
153+
}

‎src/check.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ export function check(module: Module) {
5858
return errorType;
5959
case Node.Literal:
6060
return numberType;
61+
case Node.StringLiteral:
62+
return stringType;
6163
case Node.Assignment:
6264
const v = checkExpression(expression.value);
6365
const t = checkExpression(expression.name);

‎src/emit.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,22 @@
11
import { Statement, Node, Expression } from './types';
22

3+
const singleQuoteRegex = /[\\\'\t\v\f\b\r\n]/g;
4+
const doubleQuoteRegex = /[\\\"\t\v\f\b\r\n]/g;
5+
6+
const escapedCharsMap = new Map(
7+
Object.entries({
8+
'\t': '\\t',
9+
'\v': '\\v',
10+
'\f': '\\f',
11+
'\b': '\\b',
12+
'\r': '\\r',
13+
'\n': '\\n',
14+
'\\': '\\\\',
15+
'"': '\\"',
16+
"'": "\\'",
17+
}),
18+
);
19+
320
export function emit(statements: Statement[]) {
421
return statements.map(emitStatement).join(';\n');
522
}
@@ -26,7 +43,22 @@ function emitExpression(expression: Expression): string {
2643
return expression.text;
2744
case Node.Literal:
2845
return '' + expression.value;
46+
case Node.StringLiteral:
47+
return expression.isSingleQuote
48+
? `'${escapeString(expression.value, true)}'`
49+
: `"${escapeString(expression.value, false)}"`;
2950
case Node.Assignment:
3051
return `${expression.name.text} = ${emitExpression(expression.value)}`;
3152
}
3253
}
54+
55+
function escapeString(string: string, isSingleQuote: boolean) {
56+
return string.replace(
57+
isSingleQuote ? singleQuoteRegex : doubleQuoteRegex,
58+
replacement,
59+
);
60+
}
61+
62+
function replacement(char: string) {
63+
return escapedCharsMap.get(char) || char;
64+
}

‎src/lex.ts

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Token, Lexer } from './types';
1+
import { Token, Lexer, CharCodes } from './types';
22

33
const keywords = {
44
function: Token.Function,
@@ -11,12 +11,14 @@ export function lex(s: string): Lexer {
1111
let pos = 0;
1212
let text = '';
1313
let token = Token.BOF;
14+
let firstChar: string;
1415

1516
return {
1617
scan,
1718
token: () => token,
1819
pos: () => pos,
1920
text: () => text,
21+
isSingleQuote: () => firstChar === "'",
2022
};
2123

2224
function scan() {
@@ -40,6 +42,10 @@ export function lex(s: string): Lexer {
4042
text in keywords
4143
? keywords[text as keyof typeof keywords]
4244
: Token.Identifier;
45+
} else if (['"', "'"].includes(s.charAt(pos))) {
46+
firstChar = s.charAt(pos);
47+
text = scanString();
48+
token = Token.String;
4349
} else {
4450
pos++;
4551
switch (s.charAt(pos - 1)) {
@@ -62,6 +68,64 @@ export function lex(s: string): Lexer {
6268
function scanForward(pred: (x: string) => boolean) {
6369
while (pos < s.length && pred(s.charAt(pos))) pos++;
6470
}
71+
72+
function scanString() {
73+
const quote = s.charCodeAt(pos);
74+
pos++;
75+
76+
let stringValue = '';
77+
let start = pos;
78+
79+
while (true) {
80+
if (pos >= s.length) {
81+
// report unterminated string literal error
82+
}
83+
84+
const char = s.charCodeAt(pos);
85+
86+
if (char === quote) {
87+
stringValue += s.slice(start, pos);
88+
pos++;
89+
break;
90+
}
91+
92+
if (char === CharCodes.backslash) {
93+
stringValue += s.slice(start, pos);
94+
stringValue += scanEscapeSequence();
95+
start = pos;
96+
continue;
97+
}
98+
99+
pos++;
100+
}
101+
102+
return stringValue;
103+
}
104+
105+
function scanEscapeSequence() {
106+
pos++;
107+
const char = s.charCodeAt(pos);
108+
pos++;
109+
110+
switch (char) {
111+
case CharCodes.b:
112+
return '\b';
113+
case CharCodes.t:
114+
return '\t';
115+
case CharCodes.n:
116+
return '\n';
117+
case CharCodes.r:
118+
return '\r';
119+
case CharCodes.singleQuote:
120+
// prettier-ignore
121+
return "\'";
122+
case CharCodes.doubleQuote:
123+
// prettier-ignore
124+
return '\"';
125+
default:
126+
return String.fromCharCode(char);
127+
}
128+
}
65129
}
66130

67131
export function lexAll(s: string) {

‎src/parse.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ export function parse(lexer: Lexer): Module {
3838
return { kind: Node.Identifier, text: lexer.text(), pos };
3939
} else if (tryParseToken(Token.Literal)) {
4040
return { kind: Node.Literal, value: +lexer.text(), pos };
41+
} else if (tryParseToken(Token.String)) {
42+
return {
43+
kind: Node.StringLiteral,
44+
value: lexer.text(),
45+
pos,
46+
isSingleQuote: lexer.isSingleQuote(),
47+
};
4148
}
4249
error(
4350
pos,

0 commit comments

Comments
 (0)