Skip to content

Semantic 2 fast #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: semantic_2
Choose a base branch
from
Open

Semantic 2 fast #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
adc6c9f
Initial import of the vscode semantic highlight code
orta Jun 1, 2020
fff50d6
Adds the ability to test modern semantic classification via strings i…
orta Jun 15, 2020
a95eb3d
Adds existing tests
orta Jun 16, 2020
1c1b393
Port over the semantic classification tests
orta Jun 17, 2020
de080b6
Update baselines
orta Jun 19, 2020
455ed22
Update src/harness/fourslashImpl.ts
orta Jun 22, 2020
a25badb
Handle feedback from #39119
orta Jun 22, 2020
8c9a4f5
Consistent formatting in the 2020 classifier
orta Jun 22, 2020
d425ab5
Update baselines
orta Jun 22, 2020
8849fea
Apply suggestions from code review
orta Jun 23, 2020
2a8766c
Update src/harness/fourslashImpl.ts
orta Jun 23, 2020
9c1f396
Reafactor after comments
orta Jun 23, 2020
58e830c
Switch to a faster semantic classifier
orta Jun 29, 2020
7d13c29
Gets the tests green
orta Jun 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/harness/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ namespace ts.server {
return notImplemented();
}

getEncodedSemanticClassifications(_fileName: string, _span: TextSpan): Classifications {
getEncodedSemanticClassifications(_fileName: string, _span: TextSpan, _format?: SemanticClassificationFormat): Classifications {
return notImplemented();
}

Expand Down
83 changes: 74 additions & 9 deletions src/harness/fourslashImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2448,20 +2448,64 @@ namespace FourSlash {
Harness.IO.log(this.spanInfoToString(this.getNameOrDottedNameSpan(pos)!, "**"));
}

private verifyClassifications(expected: { classificationType: string; text: string; textSpan?: TextSpan }[], actual: ts.ClassifiedSpan[], sourceFileText: string) {
private classificationToIdentifier(classification: number){

const tokenTypes: string[] = [];
tokenTypes[ts.classifier.modern.TokenType.class] = "class";
tokenTypes[ts.classifier.modern.TokenType.enum] = "enum";
tokenTypes[ts.classifier.modern.TokenType.interface] = "interface";
tokenTypes[ts.classifier.modern.TokenType.namespace] = "namespace";
tokenTypes[ts.classifier.modern.TokenType.typeParameter] = "typeParameter";
tokenTypes[ts.classifier.modern.TokenType.type] = "type";
tokenTypes[ts.classifier.modern.TokenType.parameter] = "parameter";
tokenTypes[ts.classifier.modern.TokenType.variable] = "variable";
tokenTypes[ts.classifier.modern.TokenType.enumMember] = "enumMember";
tokenTypes[ts.classifier.modern.TokenType.property] = "property";
tokenTypes[ts.classifier.modern.TokenType.function] = "function";
tokenTypes[ts.classifier.modern.TokenType.member] = "member";

const tokenModifiers: string[] = [];
tokenModifiers[ts.classifier.modern.TokenModifier.async] = "async";
tokenModifiers[ts.classifier.modern.TokenModifier.declaration] = "declaration";
tokenModifiers[ts.classifier.modern.TokenModifier.readonly] = "readonly";
tokenModifiers[ts.classifier.modern.TokenModifier.static] = "static";
tokenModifiers[ts.classifier.modern.TokenModifier.local] = "local";
tokenModifiers[ts.classifier.modern.TokenModifier.defaultLibrary] = "defaultLibrary";


function getTokenTypeFromClassification(tsClassification: number): number | undefined {
if (tsClassification > ts.classifier.modern.TokenEncodingConsts.modifierMask) {
return (tsClassification >> ts.classifier.modern.TokenEncodingConsts.typeOffset) - 1;
}
return undefined;
}

function getTokenModifierFromClassification(tsClassification: number) {
return tsClassification & ts.classifier.modern.TokenEncodingConsts.modifierMask;
}

const typeIdx = getTokenTypeFromClassification(classification) || 0;
const modSet = getTokenModifierFromClassification(classification);

return [tokenTypes[typeIdx], ...tokenModifiers.filter((_, i) => modSet & 1 << i)].join(".");
}

private verifyClassifications(expected: { classificationType: string | number, text?: string; textSpan?: TextSpan }[], actual: ts.ClassifiedSpan[], sourceFileText: string) {
if (actual.length !== expected.length) {
this.raiseError("verifyClassifications failed - expected total classifications to be " + expected.length +
", but was " + actual.length +
jsonMismatchString());
}

ts.zipWith(expected, actual, (expectedClassification, actualClassification) => {
ts.zipWith(expected, actual, (expectedClassification, actualClassification, index) => {
const expectedType = expectedClassification.classificationType;
if (expectedType !== actualClassification.classificationType) {
this.raiseError("verifyClassifications failed - expected classifications type to be " +
const actualType = typeof actualClassification.classificationType === "number" ? this.classificationToIdentifier(actualClassification.classificationType) : actualClassification.classificationType;

if (expectedType !== actualType) {
this.raiseError(`verifyClassifications failed - expected classifications at index ${index} type to be ` +
expectedType + ", but was " +
actualClassification.classificationType +
jsonMismatchString());
actualType +
displayExpectedAndActualString(JSON.stringify(expectedType, undefined, 4), JSON.stringify(actualType, undefined, 5)));
}

const expectedSpan = expectedClassification.textSpan;
Expand Down Expand Up @@ -2511,9 +2555,30 @@ namespace FourSlash {
}
}

public verifySemanticClassifications(expected: { classificationType: string; text: string }[]) {
public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
ts.createTextSpan(0, this.activeFile.content.length));
ts.createTextSpan(0, this.activeFile.content.length), format);
const replacement = [`const c2 = classification("2020");`,`verify.semanticClassificationsAre("2020",`];
for (const a of actual) {
const identifier = this.classificationToIdentifier(a.classificationType as number);
const text = this.activeFile.content.slice(a.textSpan.start, a.textSpan.start + a.textSpan.length);
replacement.push(` c2.semanticToken("${identifier}", "${text}"), `);
};
replacement.push(");");

throw new Error("You need to change the source code of fourslash test to use replaceWithSemanticClassifications");

// const fs = require("fs");
// const testfilePath = this.originalInputFileName.slice(1);
// const testfile = fs.readFileSync(testfilePath, "utf8");
// const newfile = testfile.replace("verify.replaceWithSemanticClassifications(\"2020\")", replacement.join("\n"));
// fs.writeFileSync(testfilePath, newfile);
}


public verifySemanticClassifications(format: ts.SemanticClassificationFormat, expected: { classificationType: string | number; text?: string }[]) {
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
ts.createTextSpan(0, this.activeFile.content.length), format);

this.verifyClassifications(expected, actual, this.activeFile.content);
}
Expand Down Expand Up @@ -3766,7 +3831,7 @@ namespace FourSlash {
const cancellation = new FourSlashInterface.Cancellation(state);
// eslint-disable-next-line no-eval
const f = eval(wrappedCode);
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.Classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
}
catch (err) {
// ensure 'source-map-support' is triggered while we still have the handler attached by accessing `error.stack`.
Expand Down
104 changes: 76 additions & 28 deletions src/harness/fourslashInterfaceImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,8 +504,12 @@ namespace FourSlashInterface {
/**
* This method *requires* an ordered stream of classifications for a file, and spans are highly recommended.
*/
public semanticClassificationsAre(...classifications: Classification[]) {
this.state.verifySemanticClassifications(classifications);
public semanticClassificationsAre(format: ts.SemanticClassificationFormat, ...classifications: Classification[]) {
this.state.verifySemanticClassifications(format, classifications);
}

public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
this.state.replaceWithSemanticClassifications(format);
}

public renameInfoSucceeded(displayName?: string, fullDisplayName?: string, kind?: string, kindModifiers?: string, fileToRename?: string, expectedRange?: FourSlash.Range, options?: ts.RenameInfoOptions) {
Expand Down Expand Up @@ -749,108 +753,152 @@ namespace FourSlashInterface {
}

interface Classification {
classificationType: ts.ClassificationTypeNames;
text: string;
classificationType: ts.ClassificationTypeNames | string;
text?: string;
textSpan?: FourSlash.TextSpan;
}
export namespace Classification {
export function comment(text: string, position?: number): Classification {


export function classification(format: ts.SemanticClassificationFormat) {

function semanticToken(identifier: string, text: string, _position: number): Classification {
return {
classificationType: identifier,
text
};
}

if (format === ts.SemanticClassificationFormat.TwentyTwenty) {
return {
semanticToken
};
}

function comment(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.comment, text, position);
}

export function identifier(text: string, position?: number): Classification {
function identifier(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.identifier, text, position);
}

export function keyword(text: string, position?: number): Classification {
function keyword(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.keyword, text, position);
}

export function numericLiteral(text: string, position?: number): Classification {
function numericLiteral(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.numericLiteral, text, position);
}

export function operator(text: string, position?: number): Classification {
function operator(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.operator, text, position);
}

export function stringLiteral(text: string, position?: number): Classification {
function stringLiteral(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.stringLiteral, text, position);
}

export function whiteSpace(text: string, position?: number): Classification {
function whiteSpace(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.whiteSpace, text, position);
}

export function text(text: string, position?: number): Classification {
function text(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.text, text, position);
}

export function punctuation(text: string, position?: number): Classification {
function punctuation(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.punctuation, text, position);
}

export function docCommentTagName(text: string, position?: number): Classification {
function docCommentTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.docCommentTagName, text, position);
}

export function className(text: string, position?: number): Classification {
function className(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.className, text, position);
}

export function enumName(text: string, position?: number): Classification {
function enumName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.enumName, text, position);
}

export function interfaceName(text: string, position?: number): Classification {
function interfaceName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.interfaceName, text, position);
}

export function moduleName(text: string, position?: number): Classification {
function moduleName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.moduleName, text, position);
}

export function typeParameterName(text: string, position?: number): Classification {
function typeParameterName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.typeParameterName, text, position);
}

export function parameterName(text: string, position?: number): Classification {
function parameterName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.parameterName, text, position);
}

export function typeAliasName(text: string, position?: number): Classification {
function typeAliasName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.typeAliasName, text, position);
}

export function jsxOpenTagName(text: string, position?: number): Classification {
function jsxOpenTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxOpenTagName, text, position);
}

export function jsxCloseTagName(text: string, position?: number): Classification {
function jsxCloseTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxCloseTagName, text, position);
}

export function jsxSelfClosingTagName(text: string, position?: number): Classification {
function jsxSelfClosingTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxSelfClosingTagName, text, position);
}

export function jsxAttribute(text: string, position?: number): Classification {
function jsxAttribute(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxAttribute, text, position);
}

export function jsxText(text: string, position?: number): Classification {
function jsxText(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxText, text, position);
}

export function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxAttributeStringLiteralValue, text, position);
}

function getClassification(classificationType: ts.ClassificationTypeNames, text: string, position?: number): Classification {
const textSpan = position === undefined ? undefined : { start: position, end: position + text.length };
return { classificationType, text, textSpan };
}

return {
comment,
identifier,
keyword,
numericLiteral,
operator,
stringLiteral,
whiteSpace,
text,
punctuation,
docCommentTagName,
className,
enumName,
interfaceName,
moduleName,
typeParameterName,
parameterName,
typeAliasName,
jsxOpenTagName,
jsxCloseTagName,
jsxSelfClosingTagName,
jsxAttribute,
jsxText,
jsxAttributeStringLiteralValue,
getClassification
};
}

export namespace Completion {
export import SortText = ts.Completions.SortText;
export import CompletionSource = ts.Completions.CompletionSource;
Expand Down
9 changes: 5 additions & 4 deletions src/harness/harnessLanguageService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,14 +448,15 @@ namespace Harness.LanguageService {
getSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSyntacticClassifications(fileName, span.start, span.length));
}
getSemanticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length));
getSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length, format));
}
getEncodedSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
return unwrapJSONCallResult(this.shim.getEncodedSyntacticClassifications(fileName, span.start, span.length));
}
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length));
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.Classifications {
const responseFormat = format || ts.SemanticClassificationFormat.Original;
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length, responseFormat));
}
getCompletionsAtPosition(fileName: string, position: number, preferences: ts.UserPreferences | undefined): ts.CompletionInfo {
return unwrapJSONCallResult(this.shim.getCompletionsAtPosition(fileName, position, preferences));
Expand Down
2 changes: 1 addition & 1 deletion src/server/project.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1819,7 +1819,7 @@ namespace ts.server {
packageJson.dependencies?.forEach((_, dependenyName) => addDependency(dependenyName));
packageJson.peerDependencies?.forEach((_, dependencyName) => addDependency(dependencyName));
if (dependencySelection === PackageJsonAutoImportPreference.All) {
packageJson.devDependencies?.forEach((_, dependencyName) => addDependency(dependencyName));
packageJson.devDependencies?.forEach((_gulp, dependencyName) => addDependency(dependencyName));
}
}

Expand Down
Loading