Skip to content

Commit 7a07c5c

Browse files
andrewbranchandrewbranch
committed
Implement extended data table
1 parent bd92afa commit 7a07c5c

File tree

6 files changed

+152
-70
lines changed

6 files changed

+152
-70
lines changed

‎_packages/api/src/node.ts

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,9 @@ const childProperties: Readonly<Partial<Record<SyntaxKind, readonly string[]>>>
130130
const HEADER_OFFSET_RESERVED = 0;
131131
const HEADER_OFFSET_STRING_TABLE_OFFSETS = 4;
132132
const HEADER_OFFSET_STRING_TABLE = 8;
133-
const HEADER_OFFSET_EXTENDED_DATA_OFFSETS = 12;
134-
const HEADER_OFFSET_EXTENDED_DATA = 16;
135-
const HEADER_OFFSET_NODES = 20;
136-
const HEADER_SIZE = 24;
133+
const HEADER_OFFSET_EXTENDED_DATA = 12;
134+
const HEADER_OFFSET_NODES = 16;
135+
const HEADER_SIZE = 20;
137136

138137
type NodeDataType = typeof NODE_DATA_TYPE_CHILDREN | typeof NODE_DATA_TYPE_STRING | typeof NODE_DATA_TYPE_EXTENDED;
139138
const NODE_DATA_TYPE_CHILDREN = 0x00000000;
@@ -142,6 +141,7 @@ const NODE_DATA_TYPE_EXTENDED = 0x80000000;
142141
const NODE_DATA_TYPE_MASK = 0xc0_00_00_00;
143142
const NODE_CHILD_MASK = 0x00_00_00_ff;
144143
const NODE_STRING_INDEX_MASK = 0x00_ff_ff_ff;
144+
const NODE_EXTENDED_DATA_MASK = 0x00_ff_ff_ff;
145145

146146
const NODE_OFFSET_KIND = 0;
147147
const NODE_OFFSET_POS = 4;
@@ -196,10 +196,6 @@ export class RemoteNodeBase {
196196
return this.view.getUint32(HEADER_OFFSET_STRING_TABLE, true);
197197
}
198198

199-
protected get offsetExtendedDataOffsets(): number {
200-
return this.view.getUint32(HEADER_OFFSET_EXTENDED_DATA_OFFSETS, true);
201-
}
202-
203199
protected get offsetExtendedData(): number {
204200
return this.view.getUint32(HEADER_OFFSET_EXTENDED_DATA, true);
205201
}
@@ -257,7 +253,7 @@ export class RemoteNodeList extends Array<RemoteNode> implements NodeArray<Remot
257253
}
258254

259255
private get offsetNodes(): number {
260-
return this.view.getUint32(20, true);
256+
return this.view.getUint32(HEADER_OFFSET_NODES, true);
261257
}
262258

263259
private get byteIndex(): number {
@@ -765,9 +761,28 @@ export class RemoteNode extends RemoteNodeBase implements Node {
765761
case SyntaxKind.BigIntLiteral:
766762
case SyntaxKind.RegularExpressionLiteral:
767763
case SyntaxKind.NoSubstitutionTemplateLiteral:
768-
case SyntaxKind.JSDocText:
764+
case SyntaxKind.JSDocText: {
769765
const stringIndex = this.data & NODE_STRING_INDEX_MASK;
770766
return this.getString(stringIndex);
767+
}
768+
case SyntaxKind.TemplateHead:
769+
case SyntaxKind.TemplateMiddle:
770+
case SyntaxKind.TemplateTail: {
771+
const extendedDataOffset = this.offsetExtendedData + (this.data & NODE_EXTENDED_DATA_MASK);
772+
const stringIndex = this.view.getUint32(extendedDataOffset, true);
773+
return this.getString(stringIndex);
774+
}
775+
}
776+
}
777+
778+
get rawText(): string | undefined {
779+
switch (this.kind) {
780+
case SyntaxKind.TemplateHead:
781+
case SyntaxKind.TemplateMiddle:
782+
case SyntaxKind.TemplateTail:
783+
const extendedDataOffset = this.offsetExtendedData + (this.data & NODE_EXTENDED_DATA_MASK);
784+
const stringIndex = this.view.getUint32(extendedDataOffset + 4, true);
785+
return this.getString(stringIndex);
771786
}
772787
}
773788

@@ -790,6 +805,16 @@ export class RemoteNode extends RemoteNodeBase implements Node {
790805
return SyntaxKind.WithKeyword;
791806
}
792807
}
808+
809+
get templateFlags(): number | undefined {
810+
switch (this.kind) {
811+
case SyntaxKind.TemplateHead:
812+
case SyntaxKind.TemplateMiddle:
813+
case SyntaxKind.TemplateTail:
814+
const extendedDataOffset = this.offsetExtendedData + (this.data & NODE_EXTENDED_DATA_MASK);
815+
return this.view.getUint32(extendedDataOffset + 8, true);
816+
}
817+
}
793818
}
794819

795820
export class RemoteSourceFile extends RemoteNode {

‎_packages/api/test/api.test.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
import { API } from "@typescript/api";
22
import { createVirtualFileSystem } from "@typescript/api/fs";
3+
import {
4+
isTemplateHead,
5+
isTemplateMiddle,
6+
isTemplateTail,
7+
} from "@typescript/ast";
38
import assert from "node:assert";
49
import {
510
describe,
@@ -25,4 +30,43 @@ describe("SourceFile", () => {
2530
assert.ok(sourceFile);
2631
assert.equal(sourceFile.text, files["/src/index.ts"]);
2732
});
33+
34+
test("extended data", () => {
35+
const files = {
36+
"/tsconfig.json": "{}",
37+
"/src/index.ts": "`head ${middle} tail`",
38+
};
39+
40+
const api = new API({
41+
cwd: new URL("../../../", import.meta.url).pathname,
42+
tsserverPath: new URL("../../../built/local/tsgo", import.meta.url).pathname,
43+
fs: createVirtualFileSystem(files),
44+
});
45+
46+
const project = api.loadProject("/tsconfig.json");
47+
const sourceFile = project.getSourceFile("/src/index.ts");
48+
49+
assert.ok(sourceFile);
50+
let nodeCount = 1;
51+
sourceFile.forEachChild(function visit(node) {
52+
if (isTemplateHead(node)) {
53+
assert.equal(node.text, "head ");
54+
assert.equal(node.rawText, "head ");
55+
assert.equal(node.templateFlags, 0);
56+
}
57+
else if (isTemplateMiddle(node)) {
58+
assert.equal(node.text, "middle");
59+
assert.equal(node.rawText, "middle");
60+
assert.equal(node.templateFlags, 0);
61+
}
62+
else if (isTemplateTail(node)) {
63+
assert.equal(node.text, " tail");
64+
assert.equal(node.rawText, " tail");
65+
assert.equal(node.templateFlags, 0);
66+
}
67+
nodeCount++;
68+
node.forEachChild(visit);
69+
});
70+
assert.equal(nodeCount, 7);
71+
});
2872
});

‎_packages/ast/src/nodes.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1287,19 +1287,19 @@ export type LiteralToken =
12871287
export interface TemplateHead extends TemplateLiteralLikeNode {
12881288
readonly kind: SyntaxKind.TemplateHead;
12891289
readonly parent: TemplateExpression | TemplateLiteralTypeNode;
1290-
templateFlags?: TokenFlags;
1290+
templateFlags: TokenFlags;
12911291
}
12921292

12931293
export interface TemplateMiddle extends TemplateLiteralLikeNode {
12941294
readonly kind: SyntaxKind.TemplateMiddle;
12951295
readonly parent: TemplateSpan | TemplateLiteralTypeSpan;
1296-
templateFlags?: TokenFlags;
1296+
templateFlags: TokenFlags;
12971297
}
12981298

12991299
export interface TemplateTail extends TemplateLiteralLikeNode {
13001300
readonly kind: SyntaxKind.TemplateTail;
13011301
readonly parent: TemplateSpan | TemplateLiteralTypeSpan;
1302-
templateFlags?: TokenFlags;
1302+
templateFlags: TokenFlags;
13031303
}
13041304

13051305
export type PseudoLiteralToken =

‎internal/api/encoder/encoder.go

Lines changed: 64 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ const (
2121

2222
const (
2323
NodeDataTypeChildren uint32 = iota << 30
24-
NodeDataTypeStringIndex
25-
NodeDataTypeExtendedDataIndex
24+
NodeDataTypeString
25+
NodeDataTypeExtendedData
2626
)
2727

2828
const (
@@ -36,20 +36,22 @@ const (
3636
)
3737

3838
const (
39-
HeaderOffsetReserved = iota * 4
40-
HeaderOffsetStringTableOffsets
41-
HeaderOffsetStringTable
42-
HeaderOffsetExtendedDataOffsets
39+
HeaderOffsetMetadata = iota * 4
40+
HeaderOffsetStringOffsets
41+
HeaderOffsetStringData
4342
HeaderOffsetExtendedData
4443
HeaderOffsetNodes
4544
HeaderSize
4645
)
4746

48-
// EncodeSourceFile encodes a source file into a byte slice.
47+
const (
48+
ProtocolVersion uint8 = 1
49+
)
50+
51+
// EncodeSourceFile encodes a source file into a byte slice. The protocol
4952
func EncodeSourceFile(sourceFile *ast.SourceFile) ([]byte, error) {
50-
var err error
5153
var parentIndex, nodeCount, prevIndex uint32
52-
54+
var extendedData []byte
5355
strs := newStringTable(sourceFile.Text, sourceFile.TextCount)
5456
nodes := make([]byte, 0, (sourceFile.NodeCount+1)*NodeSize)
5557

@@ -70,9 +72,7 @@ func EncodeSourceFile(sourceFile *ast.SourceFile) ([]byte, error) {
7072
nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3
7173
}
7274

73-
if nodes, err = appendUint32s(nodes, SyntaxKindNodeList, uint32(nodeList.Pos()), uint32(nodeList.End()), 0, parentIndex, uint32(len(nodeList.Nodes))); err != nil {
74-
return nil
75-
}
75+
nodes = appendUint32s(nodes, SyntaxKindNodeList, uint32(nodeList.Pos()), uint32(nodeList.End()), 0, parentIndex, uint32(len(nodeList.Nodes)))
7676

7777
saveParentIndex := parentIndex
7878

@@ -104,10 +104,7 @@ func EncodeSourceFile(sourceFile *ast.SourceFile) ([]byte, error) {
104104
nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3
105105
}
106106

107-
if nodes, err = appendUint32s(nodes, uint32(node.Kind), uint32(node.Pos()), uint32(node.End()), 0, parentIndex, getNodeData(node, strs)); err != nil {
108-
visitor.Visit = nil
109-
return nil
110-
}
107+
nodes = appendUint32s(nodes, uint32(node.Kind), uint32(node.Pos()), uint32(node.End()), 0, parentIndex, getNodeData(node, strs, &extendedData))
111108

112109
saveParentIndex := parentIndex
113110

@@ -120,70 +117,61 @@ func EncodeSourceFile(sourceFile *ast.SourceFile) ([]byte, error) {
120117
return node
121118
}
122119

123-
if nodes, err = appendUint32s(nodes, 0, 0, 0, 0, 0, 0); err != nil {
124-
return nil, err
125-
}
120+
nodes = appendUint32s(nodes, 0, 0, 0, 0, 0, 0)
126121

127122
nodeCount++
128123
parentIndex++
129-
if nodes, err = appendUint32s(nodes, uint32(sourceFile.Kind), uint32(sourceFile.Pos()), uint32(sourceFile.End()), 0, 0, 0); err != nil {
130-
return nil, err
131-
}
124+
nodes = appendUint32s(nodes, uint32(sourceFile.Kind), uint32(sourceFile.Pos()), uint32(sourceFile.End()), 0, 0, 0)
132125

133126
visitor.VisitEachChild(sourceFile.AsNode())
134-
if err != nil {
135-
return nil, err
136-
}
137127

128+
metadata := uint32(ProtocolVersion) << 24
138129
offsetStringTableOffsets := HeaderSize
139130
offsetStringTableData := HeaderSize + len(strs.offsets)*4
140-
offsetNodes := offsetStringTableData + strs.stringLength()
141-
offsetExtendedDataOffsets := 0
142-
offsetExtendedDataData := 0
131+
offsetExtendedData := offsetStringTableData + strs.stringLength()
132+
offsetNodes := offsetExtendedData + len(extendedData)
143133

144134
header := []uint32{
145-
0,
135+
metadata,
146136
uint32(offsetStringTableOffsets),
147137
uint32(offsetStringTableData),
148-
uint32(offsetExtendedDataOffsets),
149-
uint32(offsetExtendedDataData),
138+
uint32(offsetExtendedData),
150139
uint32(offsetNodes),
151140
}
152141

153142
var headerBytes, strsBytes []byte
154-
if headerBytes, err = appendUint32s(nil, header...); err != nil {
155-
return nil, err
156-
}
157-
if strsBytes, err = strs.encode(); err != nil {
158-
return nil, err
159-
}
143+
headerBytes = appendUint32s(nil, header...)
144+
strsBytes = strs.encode()
160145

161146
return slices.Concat(
162147
headerBytes,
163148
strsBytes,
149+
extendedData,
164150
nodes,
165151
), nil
166152
}
167153

168-
func appendUint32s(buf []byte, values ...uint32) ([]byte, error) {
154+
func appendUint32s(buf []byte, values ...uint32) []byte {
169155
for _, value := range values {
170156
var err error
171157
if buf, err = binary.Append(buf, binary.LittleEndian, value); err != nil {
172-
return nil, err
158+
// The only error binary.Append can return is for values that are not fixed-size.
159+
// This can never happen here, since we are always appending uint32.
160+
panic(fmt.Sprintf("failed to append uint32: %v", err))
173161
}
174162
}
175-
return buf, nil
163+
return buf
176164
}
177165

178-
func getNodeData(node *ast.Node, strs *stringTable) uint32 {
166+
func getNodeData(node *ast.Node, strs *stringTable, extendedData *[]byte) uint32 {
179167
t := getNodeDataType(node)
180168
switch t {
181169
case NodeDataTypeChildren:
182170
return t | getNodeDefinedData(node) | uint32(getChildrenPropertyMask(node))
183-
case NodeDataTypeStringIndex:
171+
case NodeDataTypeString:
184172
return t | getNodeDefinedData(node) | recordNodeStrings(node, strs)
185-
case NodeDataTypeExtendedDataIndex:
186-
return t | getNodeDefinedData(node) /* | TODO */
173+
case NodeDataTypeExtendedData:
174+
return t | getNodeDefinedData(node) | recordExtendedData(node, strs, extendedData)
187175
default:
188176
panic("unreachable")
189177
}
@@ -200,11 +188,11 @@ func getNodeDataType(node *ast.Node) uint32 {
200188
ast.KindRegularExpressionLiteral,
201189
ast.KindNoSubstitutionTemplateLiteral,
202190
ast.KindJSDocText:
203-
return NodeDataTypeStringIndex
191+
return NodeDataTypeString
204192
case ast.KindTemplateHead,
205193
ast.KindTemplateMiddle,
206194
ast.KindTemplateTail:
207-
return NodeDataTypeExtendedDataIndex
195+
return NodeDataTypeExtendedData
208196
default:
209197
return NodeDataTypeChildren
210198
}
@@ -558,9 +546,6 @@ func getNodeDefinedData(node *ast.Node) uint32 {
558546
case ast.KindImportType:
559547
n := node.AsImportTypeNode()
560548
return uint32(boolToByte(n.IsTypeOf)) << 24
561-
case ast.KindBlock:
562-
n := node.AsBlock()
563-
return uint32(boolToByte(n.Multiline)) << 24
564549
case ast.KindImportEqualsDeclaration:
565550
n := node.AsImportEqualsDeclaration()
566551
return uint32(boolToByte(n.IsTypeOnly)) << 24
@@ -570,6 +555,9 @@ func getNodeDefinedData(node *ast.Node) uint32 {
570555
case ast.KindExportDeclaration:
571556
n := node.AsExportDeclaration()
572557
return uint32(boolToByte(n.IsTypeOnly)) << 24
558+
case ast.KindBlock:
559+
n := node.AsBlock()
560+
return uint32(boolToByte(n.Multiline)) << 24
573561
case ast.KindArrayLiteralExpression:
574562
n := node.AsArrayLiteralExpression()
575563
return uint32(boolToByte(n.MultiLine)) << 24
@@ -620,6 +608,33 @@ func recordNodeStrings(node *ast.Node, strs *stringTable) uint32 {
620608
}
621609
}
622610

611+
func recordExtendedData(node *ast.Node, strs *stringTable, extendedData *[]byte) uint32 {
612+
offset := uint32(len(*extendedData))
613+
var text, rawText string
614+
var templateFlags uint32
615+
switch node.Kind {
616+
case ast.KindTemplateTail:
617+
n := node.AsTemplateTail()
618+
text = n.Text
619+
rawText = n.RawText
620+
templateFlags = uint32(n.TemplateFlags)
621+
case ast.KindTemplateMiddle:
622+
n := node.AsTemplateMiddle()
623+
text = n.Text
624+
rawText = n.RawText
625+
templateFlags = uint32(n.TemplateFlags)
626+
case ast.KindTemplateHead:
627+
n := node.AsTemplateHead()
628+
text = n.Text
629+
rawText = n.RawText
630+
templateFlags = uint32(n.TemplateFlags)
631+
}
632+
textIndex := strs.add(text, node.Kind, node.Pos(), node.End())
633+
rawTextIndex := strs.add(rawText, node.Kind, node.Pos(), node.End())
634+
*extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, templateFlags)
635+
return offset
636+
}
637+
623638
func boolToByte(b bool) byte {
624639
if b {
625640
return 1

‎internal/api/encoder/encoder_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ func formatEncodedSourceFile(encoded []byte) string {
6060
var result strings.Builder
6161
var getIndent func(parentIndex uint32) string
6262
offsetNodes := readUint32(encoded, encoder.HeaderOffsetNodes)
63-
offsetStringOffsets := readUint32(encoded, encoder.HeaderOffsetStringTableOffsets)
64-
offsetStrings := readUint32(encoded, encoder.HeaderOffsetStringTable)
63+
offsetStringOffsets := readUint32(encoded, encoder.HeaderOffsetStringOffsets)
64+
offsetStrings := readUint32(encoded, encoder.HeaderOffsetStringData)
6565
getIndent = func(parentIndex uint32) string {
6666
if parentIndex == 0 {
6767
return ""

0 commit comments

Comments
 (0)