Skip to content

Commit

Permalink
Include localized diagnostics (microsoft#18702)
Browse files Browse the repository at this point in the history
* Add lcl files

* Add loclalization script

* Add localization build targets

* use async exists, and add assert

* Generate lcg file

* Add localize task to gulpFile

* Only run loclaize if the generated files neededs update. Also run localize as part of local

* Fix lint errors

* Linter love

* Respond to code review comments
  • Loading branch information
mhegazy authored Oct 3, 2017
1 parent 3a2c723 commit a8b7f7d
Show file tree
Hide file tree
Showing 19 changed files with 110,848 additions and 17 deletions.
12 changes: 6 additions & 6 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,6 @@ tests/baselines/local/projectOutput/*
tests/baselines/reference/testresults.tap
tests/services/baselines/prototyping/local/*
tests/services/browser/typescriptServices.js
scripts/authors.js
scripts/configureNightly.js
scripts/processDiagnosticMessages.d.ts
scripts/processDiagnosticMessages.js
scripts/importDefinitelyTypedTests/importDefinitelyTypedTests.js
src/harness/*.js
src/compiler/diagnosticInformationMap.generated.ts
src/compiler/diagnosticMessages.generated.json
Expand All @@ -43,7 +38,12 @@ scripts/run.bat
scripts/word2md.js
scripts/buildProtocol.js
scripts/ior.js
scripts/buildProtocol.js
scripts/authors.js
scripts/configureNightly.js
scripts/processDiagnosticMessages.d.ts
scripts/processDiagnosticMessages.js
scripts/importDefinitelyTypedTests/importDefinitelyTypedTests.js
scripts/generateLocalizedDiagnosticMessages.js
scripts/*.js.map
scripts/typings/
coverage/
Expand Down
3 changes: 2 additions & 1 deletion .npmignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ internal
issue_template.md
jenkins.sh
lib/README.md
lib/enu
netci.groovy
pull_request_template.md
scripts
Expand All @@ -16,5 +17,5 @@ Jakefile.js
.gitattributes
.settings/
.travis.yml
.vscode/
.vscode/
test.config
44 changes: 38 additions & 6 deletions Gulpfile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ const harnessDirectory = "src/harness/";
const libraryDirectory = "src/lib/";
const scriptsDirectory = "scripts/";
const docDirectory = "doc/";
const lclDirectory = "src/loc/lcl";

const builtDirectory = "built/";
const builtLocalDirectory = "built/local/";
Expand Down Expand Up @@ -367,6 +368,36 @@ gulp.task(builtGeneratedDiagnosticMessagesJSON, [diagnosticInfoMapTs], (done) =>

gulp.task("generate-diagnostics", "Generates a diagnostic file in TypeScript based on an input JSON file", [diagnosticInfoMapTs]);

// Localize diagnostics script
const generateLocalizedDiagnosticMessagesJs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.js");
const generateLocalizedDiagnosticMessagesTs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.ts");

gulp.task(generateLocalizedDiagnosticMessagesJs, /*help*/ false, [], () => {
const settings: tsc.Settings = getCompilerSettings({
target: "es5",
declaration: false,
removeComments: true,
noResolve: false,
stripInternal: false,
types: ["node", "xml2js"]
}, /*useBuiltCompiler*/ false);
return gulp.src(generateLocalizedDiagnosticMessagesTs)
.pipe(newer(generateLocalizedDiagnosticMessagesJs))
.pipe(sourcemaps.init())
.pipe(tsc(settings))
.pipe(sourcemaps.write("."))
.pipe(gulp.dest("."));
});

// Localize diagnostics
const generatedLCGFile = path.join(builtLocalDirectory, "enu", "diagnosticMessages.generated.json.lcg");
gulp.task(generatedLCGFile, [generateLocalizedDiagnosticMessagesJs, diagnosticInfoMapTs], (done) => {
if (fs.existsSync(builtLocalDirectory) && needsUpdate(generatedDiagnosticMessagesJSON, generatedLCGFile)) {
exec(host, [generateLocalizedDiagnosticMessagesJs, lclDirectory, builtLocalDirectory, generatedDiagnosticMessagesJSON], done, done);
}
});

gulp.task("localize", [generatedLCGFile]);

const servicesFile = path.join(builtLocalDirectory, "typescriptServices.js");
const standaloneDefinitionsFile = path.join(builtLocalDirectory, "typescriptServices.d.ts");
Expand Down Expand Up @@ -494,7 +525,7 @@ gulp.task(typesMapJson, /*help*/ false, [], () => {
});

gulp.task("lssl", "Builds language service server library", [tsserverLibraryFile]);
gulp.task("local", "Builds the full compiler and services", [builtLocalCompiler, servicesFile, serverFile, builtGeneratedDiagnosticMessagesJSON, tsserverLibraryFile]);
gulp.task("local", "Builds the full compiler and services", [builtLocalCompiler, servicesFile, serverFile, builtGeneratedDiagnosticMessagesJSON, tsserverLibraryFile, "localize"]);
gulp.task("tsc", "Builds only the compiler", [builtLocalCompiler]);

// Generate Markdown spec
Expand Down Expand Up @@ -536,15 +567,16 @@ gulp.task("dontUseDebugMode", /*help*/ false, [], (done) => { useDebugMode = fal

gulp.task("VerifyLKG", /*help*/ false, [], () => {
const expectedFiles = [builtLocalCompiler, servicesFile, serverFile, nodePackageFile, nodeDefinitionsFile, standaloneDefinitionsFile, tsserverLibraryFile, tsserverLibraryDefinitionFile, typingsInstallerJs, cancellationTokenJs].concat(libraryTargets);
const missingFiles = expectedFiles.filter(function(f) {
return !fs.existsSync(f);
});
const missingFiles = expectedFiles.
concat(fs.readdirSync(lclDirectory).map(function (d) { return path.join(builtLocalDirectory, d, "diagnosticMessages.generated.json"); })).
concat(generatedLCGFile).
filter(f => !fs.existsSync(f));
if (missingFiles.length > 0) {
throw new Error("Cannot replace the LKG unless all built targets are present in directory " + builtLocalDirectory +
". The following files are missing:\n" + missingFiles.join("\n"));
}
// Copy all the targets into the LKG directory
return gulp.src(expectedFiles).pipe(gulp.dest(LKGDirectory));
return gulp.src([...expectedFiles, path.join(builtLocalDirectory, "**"), `!${path.join(builtLocalDirectory, "tslint")}`, `!${path.join(builtLocalDirectory, "*.*")}`]).pipe(gulp.dest(LKGDirectory));
});

gulp.task("LKGInternal", /*help*/ false, ["lib", "local"]);
Expand Down Expand Up @@ -1051,7 +1083,7 @@ gulp.task("lint", "Runs tslint on the compiler sources. Optional arguments are:
const fileMatcher = cmdLineOptions["files"];
const files = fileMatcher
? `src/**/${fileMatcher}`
: "Gulpfile.ts 'scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
: "Gulpfile.ts 'scripts/generateLocalizedDiagnosticMessages.ts' scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
const cmd = `node node_modules/tslint/bin/tslint ${files} --formatters-dir ./built/local/tslint/formatters --format autolinkableStylish`;
console.log("Linting: " + cmd);
child_process.execSync(cmd, { stdio: [0, 1, 2] });
Expand Down
45 changes: 41 additions & 4 deletions Jakefile.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ var libraryDirectory = "src/lib/";
var scriptsDirectory = "scripts/";
var unittestsDirectory = "src/harness/unittests/";
var docDirectory = "doc/";
var lclDirectory = "src/loc/lcl";

var builtDirectory = "built/";
var builtLocalDirectory = "built/local/";
Expand Down Expand Up @@ -427,7 +428,40 @@ compileFile(processDiagnosticMessagesJs,
[processDiagnosticMessagesTs],
[processDiagnosticMessagesTs],
[],
/*useBuiltCompiler*/ false);
/*useBuiltCompiler*/ false);

// Localize diagnostics script
var generateLocalizedDiagnosticMessagesJs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.js");
var generateLocalizedDiagnosticMessagesTs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.ts");

file(generateLocalizedDiagnosticMessagesTs);

compileFile(generateLocalizedDiagnosticMessagesJs,
[generateLocalizedDiagnosticMessagesTs],
[generateLocalizedDiagnosticMessagesTs],
[],
/*useBuiltCompiler*/ false, { noOutFile: true, types: ["node", "xml2js"] });

// Localize diagnostics
var generatedLCGFile = path.join(builtLocalDirectory, "enu", "diagnosticMessages.generated.json.lcg");
file(generatedLCGFile, [generateLocalizedDiagnosticMessagesJs, diagnosticInfoMapTs, generatedDiagnosticMessagesJSON], function () {
var cmd = host + " " + generateLocalizedDiagnosticMessagesJs + " " + lclDirectory + " " + builtLocalDirectory + " " + generatedDiagnosticMessagesJSON;
console.log(cmd);
var ex = jake.createExec([cmd]);
// Add listeners for output and error
ex.addListener("stdout", function (output) {
process.stdout.write(output);
});
ex.addListener("stderr", function (error) {
process.stderr.write(error);
});
ex.addListener("cmdEnd", function () {
complete();
});
ex.run();
}, { async: true });

task("localize", [generatedLCGFile]);

var buildProtocolTs = path.join(scriptsDirectory, "buildProtocol.ts");
var buildProtocolJs = path.join(scriptsDirectory, "buildProtocol.js");
Expand Down Expand Up @@ -644,7 +678,7 @@ task("build-fold-end", [], function () {

// Local target to build the compiler and services
desc("Builds the full compiler and services");
task("local", ["build-fold-start", "generate-diagnostics", "lib", tscFile, servicesFile, nodeDefinitionsFile, serverFile, buildProtocolDts, builtGeneratedDiagnosticMessagesJSON, "lssl", "build-fold-end"]);
task("local", ["build-fold-start", "generate-diagnostics", "lib", tscFile, servicesFile, nodeDefinitionsFile, serverFile, buildProtocolDts, builtGeneratedDiagnosticMessagesJSON, "lssl", "localize", "build-fold-end"]);

// Local target to build only tsc.js
desc("Builds only the compiler");
Expand Down Expand Up @@ -699,7 +733,10 @@ task("generate-spec", [specMd]);
// Makes a new LKG. This target does not build anything, but errors if not all the outputs are present in the built/local directory
desc("Makes a new LKG out of the built js files");
task("LKG", ["clean", "release", "local"].concat(libraryTargets), function () {
var expectedFiles = [tscFile, servicesFile, serverFile, nodePackageFile, nodeDefinitionsFile, standaloneDefinitionsFile, tsserverLibraryFile, tsserverLibraryDefinitionFile, cancellationTokenFile, typingsInstallerFile, buildProtocolDts, watchGuardFile].concat(libraryTargets);
var expectedFiles = [tscFile, servicesFile, serverFile, nodePackageFile, nodeDefinitionsFile, standaloneDefinitionsFile, tsserverLibraryFile, tsserverLibraryDefinitionFile, cancellationTokenFile, typingsInstallerFile, buildProtocolDts, watchGuardFile].
concat(libraryTargets).
concat(fs.readdirSync(lclDirectory).map(function (d) { return path.join(builtLocalDirectory, d) })).
concat(path.dirname(generatedLCGFile));
var missingFiles = expectedFiles.filter(function (f) {
return !fs.existsSync(f);
});
Expand Down Expand Up @@ -1229,7 +1266,7 @@ task("lint", ["build-rules"], () => {
const fileMatcher = process.env.f || process.env.file || process.env.files;
const files = fileMatcher
? `src/**/${fileMatcher}`
: "Gulpfile.ts 'scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
: "Gulpfile.ts 'scripts/generateLocalizedDiagnosticMessages.ts' 'scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
const cmd = `node node_modules/tslint/bin/tslint ${files} --formatters-dir ./built/local/tslint/formatters --format autolinkableStylish`;
console.log("Linting: " + cmd);
jake.exec([cmd], { interactive: true }, () => {
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
"@types/q": "latest",
"@types/run-sequence": "latest",
"@types/through2": "latest",
"@types/xml2js": "^0.4.0",
"xml2js": "^0.4.19",
"browser-resolve": "^1.11.2",
"browserify": "latest",
"chai": "latest",
Expand Down
139 changes: 139 additions & 0 deletions scripts/generateLocalizedDiagnosticMessages.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
import * as fs from "fs";
import * as path from "path";
import * as xml2js from "xml2js";

function main(): void {
const args = process.argv.slice(2);
if (args.length !== 3) {
console.log("Usage:");
console.log("\tnode generateLocalizedDiagnosticMessages.js <lcl source directory> <output directory> <generated diagnostics map file>");
return;
}

const inputPath = args[0];
const outputPath = args[1];
const diagnosticsMapFilePath = args[2];

// generate the lcg file for enu
generateLCGFile();

// generate other langs
fs.readdir(inputPath, (err, files) => {
handleError(err);
files.forEach(visitDirectory);
});

return;

function visitDirectory(name: string) {
const inputFilePath = path.join(inputPath, name, "diagnosticMessages", "diagnosticMessages.generated.json.lcl");
const outputFilePath = path.join(outputPath, name, "diagnosticMessages.generated.json");
fs.readFile(inputFilePath, (err, data) => {
handleError(err);
xml2js.parseString(data.toString(), (err, result) => {
handleError(err);
writeFile(outputFilePath, xmlObjectToString(result));
});
});
}

function handleError(err: null | object) {
if (err) {
console.error(err);
process.exit(1);
}
}

function xmlObjectToString(o: any) {
const out: any = {};
for (const item of o["LCX"]["Item"][0]["Item"][0]["Item"]) {
let ItemId = item["$"]["ItemId"];
let Val = item["Str"][0]["Tgt"] ? item["Str"][0]["Tgt"][0]["Val"][0] : item["Str"][0]["Val"][0];

if (typeof ItemId !== "string" || typeof Val !== "string") {
console.error("Unexpected XML file structure");
process.exit(1);
}

if (ItemId.charAt(0) === ";") {
ItemId = ItemId.slice(1); // remove leading semicolon
}

Val = Val.replace(/]5D;/, "]"); // unescape `]`
out[ItemId] = Val;
}
return JSON.stringify(out, undefined, 2);
}


function ensureDirectoryExists(directoryPath: string, action: () => void) {
fs.exists(directoryPath, exists => {
if (!exists) {
const basePath = path.dirname(directoryPath);
if (basePath !== directoryPath) {
return ensureDirectoryExists(basePath, () => fs.mkdir(directoryPath, action));
}
}
action();
});
}

function writeFile(fileName: string, contents: string) {
ensureDirectoryExists(path.dirname(fileName), () => {
fs.writeFile(fileName, contents, handleError);
});
}

function objectToList(o: Record<string, string>) {
const list: { key: string, value: string }[] = [];
for (const key in o) {
list.push({ key, value: o[key] });
}
return list;
}

function generateLCGFile() {
return fs.readFile(diagnosticsMapFilePath, (err, data) => {
handleError(err);
writeFile(
path.join(outputPath, "enu", "diagnosticMessages.generated.json.lcg"),
getLCGFileXML(
objectToList(JSON.parse(data.toString()))
.sort((a, b) => a.key > b.key ? 1 : -1) // lcg sorted by property keys
.reduce((s, { key, value }) => s + getItemXML(key, value), "")
));
});

function getItemXML(key: string, value: string) {
// escape entrt value
value = value.replace(/]/, "]5D;");

return `
<Item ItemId=";${key}" ItemType="0" PsrId="306" Leaf="true">
<Str Cat="Text">
<Val><![CDATA[${value}]]></Val>
</Str>
<Disp Icon="Str" />
</Item>`;
}

function getLCGFileXML(items: string) {
return `<?xml version="1.0" encoding="utf-8"?>
<LCX SchemaVersion="6.0" Name="diagnosticMessages.generated.json" PsrId="306" FileType="1" SrcCul="en-US" xmlns="http://schemas.microsoft.com/locstudio/2006/6/lcx">
<OwnedComments>
<Cmt Name="Dev" />
<Cmt Name="LcxAdmin" />
<Cmt Name="Rccx" />
</OwnedComments>
<Item ItemId=";String Table" ItemType="0" PsrId="306" Leaf="false">
<Disp Icon="Expand" Expand="true" Disp="true" LocTbl="false" />
<Item ItemId=";Strings" ItemType="0" PsrId="306" Leaf="false">
<Disp Icon="Str" Disp="true" LocTbl="false" />${items}
</Item>
</Item>
</LCX>`;
}
}
}

main();
Loading

0 comments on commit a8b7f7d

Please sign in to comment.