diff --git a/.chronus/changes/source-loader-2024-8-3-19-23-57.md b/.chronus/changes/source-loader-2024-8-3-19-23-57.md new file mode 100644 index 0000000000..4fc7489e4d --- /dev/null +++ b/.chronus/changes/source-loader-2024-8-3-19-23-57.md @@ -0,0 +1,8 @@ +--- +# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking +changeKind: fix +packages: + - "@typespec/compiler" +--- + +API: Extract source resolution logic into its own source loader diff --git a/packages/compiler/src/core/program.ts b/packages/compiler/src/core/program.ts index 769a259bda..c0ec4f9fd8 100644 --- a/packages/compiler/src/core/program.ts +++ b/packages/compiler/src/core/program.ts @@ -2,23 +2,12 @@ import { EmitterOptions } from "../config/types.js"; import { createAssetEmitter } from "../emitter-framework/asset-emitter.js"; import { validateEncodedNamesConflicts } from "../lib/encoded-names.js"; import { MANIFEST } from "../manifest.js"; -import { - deepEquals, - doIO, - findProjectRoot, - isDefined, - mapEquals, - mutate, - resolveTspMain, -} from "../utils/misc.js"; +import { deepEquals, findProjectRoot, isDefined, mapEquals, mutate } from "../utils/misc.js"; import { createBinder } from "./binder.js"; import { Checker, createChecker } from "./checker.js"; import { createSuppressCodeFix } from "./compiler-code-fixes/suppress.codefix.js"; import { compilerAssert } from "./diagnostics.js"; -import { - resolveTypeSpecEntrypoint, - resolveTypeSpecEntrypointForDir, -} from "./entrypoint-resolution.js"; +import { resolveTypeSpecEntrypoint } from "./entrypoint-resolution.js"; import { ExternalError } from "./external-error.js"; import { getLibraryUrlsLoaded } from "./library.js"; import { createLinter, resolveLinterDefinition } from "./linter.js"; @@ -33,15 +22,14 @@ import { resolveModule, } from "./module-resolver.js"; import { CompilerOptions } from "./options.js"; -import { isImportStatement, parse, parseStandaloneTypeReference } from "./parser.js"; +import { parse, parseStandaloneTypeReference } from "./parser.js"; import { getDirectoryPath, joinPaths, resolvePath } from "./path-utils.js"; import { createProjector } from "./projector.js"; -import { createSourceFile } from "./source-file.js"; +import { SourceLoader, SourceResolution, createSourceLoader, loadJsFile } from "./source-loader.js"; import { StateMap, StateSet, createStateAccessors } from "./state-accessors.js"; import { CompilerHost, Diagnostic, - DiagnosticTarget, Directive, DirectiveExpressionNode, EmitContext, @@ -56,7 +44,6 @@ import { Namespace, NoTarget, Node, - NodeFlags, ProjectionApplication, Projector, SourceFile, @@ -150,12 +137,10 @@ export async function compile( const stateMaps = new Map(); const stateSets = new Map(); const diagnostics: Diagnostic[] = []; - const seenSourceFiles = new Set(); const duplicateSymbols = new Set(); const emitters: EmitterRef[] = []; const requireImports = new Map(); - const loadedLibraries = new Map(); - const sourceFileLocationContexts = new WeakMap(); + let sourceResolution: SourceResolution; let error = false; let continueToNextStage = true; @@ -201,27 +186,12 @@ export async function compile( } const binder = createBinder(program); - await loadIntrinsicTypes(); - if (!options?.nostdlib) { - await loadStandardLibrary(); - } - - // Load additional imports prior to compilation - if (resolvedMain && options.additionalImports) { - const importScript = options.additionalImports.map((i) => `import "${i}";`).join("\n"); - const sourceFile = createSourceFile( - importScript, - joinPaths(getDirectoryPath(resolvedMain), `__additional_imports`) - ); - sourceFileLocationContexts.set(sourceFile, { type: "project" }); - await loadTypeSpecScript(sourceFile); - } - - if (resolvedMain) { - await loadMain(resolvedMain); - } else { + if (resolvedMain === undefined) { return program; } + await checkForCompilerVersionMismatch(resolvedMain); + + await loadSources(resolvedMain); const basedir = getDirectoryPath(resolvedMain); @@ -292,7 +262,7 @@ export async function compile( } } - const libraries = new Map([...loadedLibraries.entries()]); + const libraries = new Map([...sourceResolution.loadedLibraries.entries()]); const incompatibleLibraries = new Map(); for (const root of loadedRoots) { const packageJsonPath = joinPaths(root, "package.json"); @@ -328,105 +298,58 @@ export async function compile( } } - async function loadIntrinsicTypes() { - const locationContext: LocationContext = { type: "compiler" }; - await loadTypeSpecFile( - resolvePath(host.getExecutionRoot(), "lib/intrinsics.tsp"), - locationContext, - NoTarget - ); - } - - async function loadStandardLibrary() { - const locationContext: LocationContext = { type: "compiler" }; - for (const dir of host.getLibDirs()) { - await loadDirectory(dir, locationContext, NoTarget); - } - } + async function loadSources(entrypoint: string) { + const sourceLoader = await createSourceLoader(host, { + parseOptions: options.parseOptions, + getCachedScript: (file) => + oldProgram?.sourceFiles.get(file.path) ?? host.parseCache?.get(file), + }); - async function loadDirectory( - dir: string, - locationContext: LocationContext, - diagnosticTarget: DiagnosticTarget | typeof NoTarget - ): Promise { - const mainFile = await resolveTypeSpecEntrypointForDir(host, dir, reportDiagnostic); - await loadTypeSpecFile(mainFile, locationContext, diagnosticTarget); - return mainFile; - } + // intrinsic.tsp + await loadIntrinsicTypes(sourceLoader); - async function loadTypeSpecFile( - path: string, - locationContext: LocationContext, - diagnosticTarget: DiagnosticTarget | typeof NoTarget - ) { - if (seenSourceFiles.has(path)) { - return; + // standard library + if (!options?.nostdlib) { + await loadStandardLibrary(sourceLoader); } - seenSourceFiles.add(path); - const file = await doIO(host.readFile, path, program.reportDiagnostic, { - diagnosticTarget, - }); + // main entrypoint + await sourceLoader.importFile(entrypoint, { type: "project" }, "entrypoint"); - if (file) { - sourceFileLocationContexts.set(file, locationContext); - await loadTypeSpecScript(file); + // additional imports + for (const additionalImport of options?.additionalImports ?? []) { + await sourceLoader.importPath(additionalImport, NoTarget, getDirectoryPath(entrypoint), { + type: "project", + }); } - } - async function loadJsFile( - path: string, - locationContext: LocationContext, - diagnosticTarget: DiagnosticTarget | typeof NoTarget - ): Promise { - const sourceFile = program.jsSourceFiles.get(path); - if (sourceFile !== undefined) { - return sourceFile; - } + sourceResolution = sourceLoader.resolution; - const file = createSourceFile("", path); - sourceFileLocationContexts.set(file, locationContext); - const exports = await doIO(host.getJsImport, path, program.reportDiagnostic, { - diagnosticTarget, - jsDiagnosticTarget: { file, pos: 0, end: 0 }, - }); + program.sourceFiles = sourceResolution.sourceFiles; + program.jsSourceFiles = sourceResolution.jsSourceFiles; - if (!exports) { - return undefined; + // Bind + for (const file of sourceResolution.sourceFiles.values()) { + binder.bindSourceFile(file); } + for (const jsFile of sourceResolution.jsSourceFiles.values()) { + binder.bindJsSourceFile(jsFile); + } + program.reportDiagnostics(sourceResolution.diagnostics); + } - return { - kind: SyntaxKind.JsSourceFile, - id: { - kind: SyntaxKind.Identifier, - sv: "", - pos: 0, - end: 0, - symbol: undefined!, - flags: NodeFlags.Synthetic, - }, - esmExports: exports, - file, - namespaceSymbols: [], - symbol: undefined!, - pos: 0, - end: 0, - flags: NodeFlags.None, - }; + async function loadIntrinsicTypes(loader: SourceLoader) { + const locationContext: LocationContext = { type: "compiler" }; + return loader.importFile( + resolvePath(host.getExecutionRoot(), "lib/intrinsics.tsp"), + locationContext + ); } - /** - * Import the Javascript files decorator and lifecycle hooks. - */ - async function importJsFile( - path: string, - locationContext: LocationContext, - diagnosticTarget: DiagnosticTarget | typeof NoTarget - ) { - const file = await loadJsFile(path, locationContext, diagnosticTarget); - if (file !== undefined) { - program.jsSourceFiles.set(path, file); - binder.bindJsSourceFile(file); + async function loadStandardLibrary(loader: SourceLoader) { + const locationContext: LocationContext = { type: "compiler" }; + for (const dir of host.getLibDirs()) { + await loader.importFile(resolvePath(dir, "main.tsp"), locationContext); } } @@ -441,7 +364,6 @@ export async function compile( program.reportDiagnostics(script.parseDiagnostics); program.sourceFiles.set(file.path, script); binder.bindSourceFile(script); - await loadScriptImports(script); return script; } @@ -455,75 +377,12 @@ export async function compile( return script; } - async function loadScriptImports(file: TypeSpecScriptNode) { - // collect imports - const basedir = getDirectoryPath(file.file.path); - await loadImports( - file.statements.filter(isImportStatement).map((x) => ({ path: x.path.value, target: x })), - basedir, - getSourceFileLocationContext(file.file) - ); - } - function getSourceFileLocationContext(sourcefile: SourceFile): LocationContext { - const locationContext = sourceFileLocationContexts.get(sourcefile); + const locationContext = sourceResolution.locationContexts.get(sourcefile); compilerAssert(locationContext, "SourceFile should have a declaration locationContext."); return locationContext; } - async function loadImports( - imports: Array<{ path: string; target: DiagnosticTarget | typeof NoTarget }>, - relativeTo: string, - locationContext: LocationContext - ) { - // collect imports - for (const { path, target } of imports) { - await loadImport(path, target, relativeTo, locationContext); - } - } - - async function loadImport( - path: string, - target: DiagnosticTarget | typeof NoTarget, - relativeTo: string, - locationContext: LocationContext - ) { - const library = await resolveTypeSpecLibrary(path, relativeTo, target); - if (library === undefined) { - return; - } - if (library.type === "module") { - loadedLibraries.set(library.manifest.name, { - path: library.path, - manifest: library.manifest, - }); - trace("import-resolution.library", `Loading library "${path}" from "${library.mainFile}"`); - - const metadata = computeModuleMetadata(library); - locationContext = { - type: "library", - metadata, - }; - } - const importFilePath = library.type === "module" ? library.mainFile : library.path; - - const isDirectory = (await host.stat(importFilePath)).isDirectory(); - if (isDirectory) { - return await loadDirectory(importFilePath, locationContext, target); - } - - const sourceFileKind = host.getSourceFileKind(importFilePath); - - switch (sourceFileKind) { - case "js": - return await importJsFile(importFilePath, locationContext, target); - case "typespec": - return await loadTypeSpecFile(importFilePath, locationContext, target); - default: - program.reportDiagnostic(createDiagnostic({ code: "invalid-import", target })); - } - } - async function loadEmitters( basedir: string, emitterNameOrPaths: string[], @@ -558,9 +417,9 @@ export async function compile( } const entrypoint = module.type === "file" ? module.path : module.mainFile; - const file = await loadJsFile(entrypoint, locationContext, NoTarget); + const [file, jsDiagnostics] = await loadJsFile(host, entrypoint, NoTarget); - return [{ module, entrypoint: file }, []]; + return [{ module, entrypoint: file }, jsDiagnostics]; } async function loadLibrary( @@ -746,7 +605,7 @@ export async function compile( function validateRequiredImports() { for (const [requiredImport, emitterName] of requireImports) { - if (!loadedLibraries.has(requiredImport)) { + if (!sourceResolution.loadedLibraries.has(requiredImport)) { program.reportDiagnostic( createDiagnostic({ code: "missing-import", @@ -758,47 +617,6 @@ export async function compile( } } - /** - * resolves a module specifier like "myLib" to an absolute path where we can find the main of - * that module, e.g. "/typespec/node_modules/myLib/main.tsp". - */ - async function resolveTypeSpecLibrary( - specifier: string, - baseDir: string, - target: DiagnosticTarget | typeof NoTarget - ): Promise { - try { - return await resolveModule(getResolveModuleHost(), specifier, { - baseDir, - directoryIndexFiles: ["main.tsp", "index.mjs", "index.js"], - resolveMain(pkg) { - // this lets us follow node resolve semantics more-or-less exactly - // but using tspMain instead of main. - return resolveTspMain(pkg) ?? pkg.main; - }, - }); - } catch (e: any) { - if (e.code === "MODULE_NOT_FOUND") { - program.reportDiagnostic( - createDiagnostic({ code: "import-not-found", format: { path: specifier }, target }) - ); - return undefined; - } else if (e.code === "INVALID_MAIN") { - program.reportDiagnostic( - createDiagnostic({ - code: "library-invalid", - format: { path: specifier }, - messageId: "tspMain", - target, - }) - ); - return undefined; - } else { - throw e; - } - } - } - /** * resolves a module specifier like "myLib" to an absolute path where we can find the main of * that module, e.g. "/typespec/node_modules/myLib/dist/lib.js". @@ -850,26 +668,6 @@ export async function compile( }; } - /** - * Load the main file from the given path - * @param mainPath Absolute path to the main file. - */ - async function loadMain(mainPath: string): Promise { - await checkForCompilerVersionMismatch(mainPath); - - const sourceFileKind = host.getSourceFileKind(mainPath); - - const locationContext: LocationContext = { type: "project" }; - switch (sourceFileKind) { - case "js": - return await importJsFile(mainPath, locationContext, NoTarget); - case "typespec": - return await loadTypeSpecFile(mainPath, locationContext, NoTarget); - default: - program.reportDiagnostic(createDiagnostic({ code: "invalid-main", target: NoTarget })); - } - } - // It's important that we use the compiler version that resolves locally // from the input TypeSpec source location. Otherwise, there will be undefined // runtime behavior when decorators and handlers expect a diff --git a/packages/compiler/src/core/source-loader.ts b/packages/compiler/src/core/source-loader.ts new file mode 100644 index 0000000000..0438c68b23 --- /dev/null +++ b/packages/compiler/src/core/source-loader.ts @@ -0,0 +1,374 @@ +import { deepEquals, doIO, resolveTspMain } from "../utils/misc.js"; +import { compilerAssert, createDiagnosticCollector } from "./diagnostics.js"; +import { resolveTypeSpecEntrypointForDir } from "./entrypoint-resolution.js"; +import { createDiagnostic } from "./messages.js"; +import { + ModuleResolutionResult, + NodePackage, + ResolvedModule, + resolveModule, + ResolveModuleHost, +} from "./module-resolver.js"; +import { isImportStatement, parse } from "./parser.js"; +import { getDirectoryPath } from "./path-utils.js"; +import { createSourceFile } from "./source-file.js"; +import { + DiagnosticTarget, + ModuleLibraryMetadata, + NodeFlags, + NoTarget, + ParseOptions, + SourceFile, + SyntaxKind, + Tracer, + type CompilerHost, + type Diagnostic, + type JsSourceFileNode, + type LocationContext, + type TypeSpecScriptNode, +} from "./types.js"; + +export interface SourceResolution { + /** TypeSpec source files */ + readonly sourceFiles: Map; + + /** Javascript source files(Entrypoint only) */ + readonly jsSourceFiles: Map; + + readonly locationContexts: WeakMap; + readonly loadedLibraries: Map; + + readonly diagnostics: readonly Diagnostic[]; +} + +interface TypeSpecLibraryReference { + path: string; + manifest: NodePackage; +} + +export interface LoadSourceOptions { + readonly parseOptions?: ParseOptions; + readonly tracer?: Tracer; + getCachedScript?: (file: SourceFile) => TypeSpecScriptNode | undefined; +} + +export interface SourceLoader { + importFile( + path: string, + locationContext?: LocationContext, + kind?: "import" | "entrypoint" + ): Promise; + importPath( + path: string, + target: DiagnosticTarget | typeof NoTarget, + relativeTo: string, + locationContext?: LocationContext + ): Promise; + readonly resolution: SourceResolution; +} + +/** + * Create a TypeSpec source loader. This will be able to resolve and load TypeSpec and JS files. + * @param host Compiler host + * @param options Loading options + */ +export async function createSourceLoader( + host: CompilerHost, + options?: LoadSourceOptions +): Promise { + const diagnostics = createDiagnosticCollector(); + const tracer = options?.tracer; + const seenSourceFiles = new Set(); + const sourceFileLocationContexts = new WeakMap(); + const sourceFiles = new Map(); + const jsSourceFiles = new Map(); + const loadedLibraries = new Map(); + + async function importFile( + path: string, + locationContext: LocationContext, + kind: "import" | "entrypoint" = "import" + ) { + const sourceFileKind = host.getSourceFileKind(path); + + switch (sourceFileKind) { + case "js": + await importJsFile(path, locationContext, NoTarget); + break; + case "typespec": + await loadTypeSpecFile(path, locationContext, NoTarget); + break; + default: + diagnostics.add( + createDiagnostic({ + code: kind === "import" ? "invalid-import" : "invalid-main", + target: NoTarget, + }) + ); + } + } + + return { + importFile, + importPath, + resolution: { + sourceFiles, + jsSourceFiles, + locationContexts: sourceFileLocationContexts, + loadedLibraries: loadedLibraries, + diagnostics: diagnostics.diagnostics, + }, + }; + + async function loadTypeSpecFile( + path: string, + locationContext: LocationContext, + diagnosticTarget: DiagnosticTarget | typeof NoTarget + ) { + if (seenSourceFiles.has(path)) { + return; + } + seenSourceFiles.add(path); + + const file = await doIO(host.readFile, path, (x) => diagnostics.add(x), { + diagnosticTarget, + }); + + if (file) { + sourceFileLocationContexts.set(file, locationContext); + await loadTypeSpecScript(file); + } + } + + async function loadTypeSpecScript(file: SourceFile): Promise { + // This is not a diagnostic because the compiler should never reuse the same path. + // It's the caller's responsibility to use unique paths. + if (sourceFiles.has(file.path)) { + throw new RangeError("Duplicate script path: " + file.path); + } + + const script = parseOrReuse(file); + for (const diagnostic of script.parseDiagnostics) { + diagnostics.add(diagnostic); + } + + sourceFiles.set(file.path, script); + await loadScriptImports(script); + return script; + } + + function parseOrReuse(file: SourceFile): TypeSpecScriptNode { + if (options?.getCachedScript) { + const old = options.getCachedScript(file); + if (old?.file === file && deepEquals(old.parseOptions, options.parseOptions)) { + return old; + } + } + const script = parse(file, options?.parseOptions); + host.parseCache?.set(file, script); + return script; + } + + async function loadScriptImports(file: TypeSpecScriptNode) { + // collect imports + const basedir = getDirectoryPath(file.file.path); + await loadImports( + file.statements.filter(isImportStatement).map((x) => ({ path: x.path.value, target: x })), + basedir, + getSourceFileLocationContext(file.file) + ); + } + + function getSourceFileLocationContext(sourcefile: SourceFile): LocationContext { + const locationContext = sourceFileLocationContexts.get(sourcefile); + compilerAssert(locationContext, "SourceFile should have a declaration locationContext."); + return locationContext; + } + + async function loadImports( + imports: Array<{ path: string; target: DiagnosticTarget | typeof NoTarget }>, + relativeTo: string, + locationContext: LocationContext + ) { + // collect imports + for (const { path, target } of imports) { + await importPath(path, target, relativeTo, locationContext); + } + } + + async function importPath( + path: string, + target: DiagnosticTarget | typeof NoTarget, + relativeTo: string, + locationContext: LocationContext = { type: "project" } + ) { + const library = await resolveTypeSpecLibrary(path, relativeTo, target); + if (library === undefined) { + return; + } + if (library.type === "module") { + loadedLibraries.set(library.manifest.name, { + path: library.path, + manifest: library.manifest, + }); + tracer?.trace( + "import-resolution.library", + `Loading library "${path}" from "${library.mainFile}"` + ); + + const metadata = computeModuleMetadata(library); + locationContext = { + type: "library", + metadata, + }; + } + const importFilePath = library.type === "module" ? library.mainFile : library.path; + + const isDirectory = (await host.stat(importFilePath)).isDirectory(); + if (isDirectory) { + await loadDirectory(importFilePath, locationContext, target); + return; + } + + return importFile(importFilePath, locationContext); + } + + /** + * resolves a module specifier like "myLib" to an absolute path where we can find the main of + * that module, e.g. "/typespec/node_modules/myLib/main.tsp". + */ + async function resolveTypeSpecLibrary( + specifier: string, + baseDir: string, + target: DiagnosticTarget | typeof NoTarget + ): Promise { + try { + return await resolveModule(getResolveModuleHost(), specifier, { + baseDir, + directoryIndexFiles: ["main.tsp", "index.mjs", "index.js"], + resolveMain(pkg) { + // this lets us follow node resolve semantics more-or-less exactly + // but using tspMain instead of main. + return resolveTspMain(pkg) ?? pkg.main; + }, + }); + } catch (e: any) { + if (e.code === "MODULE_NOT_FOUND") { + diagnostics.add( + createDiagnostic({ code: "import-not-found", format: { path: specifier }, target }) + ); + return undefined; + } else if (e.code === "INVALID_MAIN") { + diagnostics.add( + createDiagnostic({ + code: "library-invalid", + format: { path: specifier }, + messageId: "tspMain", + target, + }) + ); + return undefined; + } else { + throw e; + } + } + } + + async function loadDirectory( + dir: string, + locationContext: LocationContext, + diagnosticTarget: DiagnosticTarget | typeof NoTarget + ): Promise { + const mainFile = await resolveTypeSpecEntrypointForDir(host, dir, (x) => diagnostics.add(x)); + await loadTypeSpecFile(mainFile, locationContext, diagnosticTarget); + return mainFile; + } + + /** + * Import the Javascript files decorator and lifecycle hooks. + */ + async function importJsFile( + path: string, + locationContext: LocationContext, + diagnosticTarget: DiagnosticTarget | typeof NoTarget + ) { + const sourceFile = jsSourceFiles.get(path); + if (sourceFile !== undefined) { + return sourceFile; + } + + const file = diagnostics.pipe(await loadJsFile(host, path, diagnosticTarget)); + if (file !== undefined) { + sourceFileLocationContexts.set(file.file, locationContext); + jsSourceFiles.set(path, file); + } + return file; + } + + function getResolveModuleHost(): ResolveModuleHost { + return { + realpath: host.realpath, + stat: host.stat, + readFile: async (path) => { + const file = await host.readFile(path); + return file.text; + }, + }; + } +} + +function computeModuleMetadata(module: ResolvedModule): ModuleLibraryMetadata { + const metadata: ModuleLibraryMetadata = { + type: "module", + name: module.manifest.name, + }; + + if (module.manifest.homepage) { + metadata.homepage = module.manifest.homepage; + } + if (module.manifest.bugs?.url) { + metadata.bugs = { url: module.manifest.bugs?.url }; + } + if (module.manifest.version) { + metadata.version = module.manifest.version; + } + + return metadata; +} + +export async function loadJsFile( + host: CompilerHost, + path: string, + diagnosticTarget: DiagnosticTarget | typeof NoTarget +): Promise<[JsSourceFileNode | undefined, readonly Diagnostic[]]> { + const file = createSourceFile("", path); + const diagnostics: Diagnostic[] = []; + const exports = await doIO(host.getJsImport, path, (x) => diagnostics.push(x), { + diagnosticTarget, + jsDiagnosticTarget: { file, pos: 0, end: 0 }, + }); + + if (!exports) { + return [undefined, diagnostics]; + } + + const node: JsSourceFileNode = { + kind: SyntaxKind.JsSourceFile, + id: { + kind: SyntaxKind.Identifier, + sv: "", + pos: 0, + end: 0, + symbol: undefined!, + flags: NodeFlags.Synthetic, + }, + esmExports: exports, + file, + namespaceSymbols: [], + symbol: undefined!, + pos: 0, + end: 0, + flags: NodeFlags.None, + }; + return [node, diagnostics]; +}