fix(deducer): failed to bundle the dependencies of pyright-internal when publishing pyright-deducer

.changeset/afraid-beds-drum.md

@@ -0,0 +1,10 @@
+---
+"@plutolang/pyright-deducer": patch
+"@plutolang/cli": patch
+---
+
+fix(deducer): failed to bundle the dependencies of pyright-internal when publishing pyright-deducer
+
+Before, we used the `bundleDependencies` option to bundle the dependencies of `pyright-internal` when publishing `pyright-deducer`. However, it failed to include the dependencies of `pyright-internal` when publishing `pyright-deducer`. So, we opted to utilize webpack to bundle the entire `pyright-deducer` package along with all its dependencies, including `pyright-internal` and its dependencies.
+
+Because webpack bundles each dependency of `pyright-deducer` into a single file, if we attempt to verify whether an instance of PyrightDeducer from `pyright-deducer` is an instance of Deducer from `@plutolang/base`, we will receive a false result. Therefore, we should check for the existence of the `deduce` method instead.

apps/cli/src/commands/utils.ts

@@ -28,12 +28,19 @@ export async function buildDeducer(
   deducerArgs: core.NewDeducerArgs
 ): Promise<core.Deducer> {
   const deducer = new (await loadPackage(deducerPkg))(deducerArgs);
-  if (deducer instanceof core.Deducer) {
+  if (isDeducer(deducer)) {
     return deducer;
   }
   throw new Error(`The default export of '${deducerPkg}' package is not a valid Deducer.`);
 }
 
+function isDeducer(obj: any): obj is core.Deducer {
+  // Because the deducer package could be bundled by webpack, when trying to confirm if an object is
+  // an instance of core.Deducer, we might get a false. Hence, we simply verify the presence of the
+  // `deduce` method instead.
+  return typeof obj.deduce === "function";
+}
+
 export async function buildGenerator(
   generatorPkg: string,
   generatorArgs: core.NewGeneratorArgs

components/deducers/python-pyright/.gitignore

@@ -1,2 +1,3 @@
-**/dist/
-libs/pyright-internal/
+dist/
+libs/pyright-internal/
+out/

components/deducers/python-pyright/__mocks__/txtMock.js

@@ -0,0 +1,7 @@
+module.exports = {
+  process(sourceText) {
+    return {
+      code: `module.exports = ${JSON.stringify(sourceText)};`,
+    };
+  },
+};

components/deducers/python-pyright/jest.config.js

@@ -20,6 +20,7 @@ module.exports = {
         },
       },
     ],
+    "\\.txt$": "<rootDir>/__mocks__/txtMock.js",
   },
   testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$",
   moduleFileExtensions: ["ts", "tsx", "js", "jsx"],

components/deducers/python-pyright/package.json

@@ -7,18 +7,17 @@
     "directory": "components/deducers/python-pyright"
   },
   "author": "plutolang",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
+  "main": "dist/bundle.js",
   "files": [
     "dist"
   ],
   "scripts": {
-    "build": "bash scripts/prepare-pyright-pkg.sh && node scripts/copy-assets.js && tsc",
-    "watch": "npm run build -- --watch",
+    "build": "bash scripts/prepare-pyright-pkg.sh && webpack --mode=production",
+    "watch": "bash scripts/prepare-pyright-pkg.sh && webpack --mode=development --watch",
     "test": "jest --verbose --coverage",
     "test:watch": "jest --verbose --coverage --watch",
     "lint": "eslint .",
-    "clean": "tsc --build --clean"
+    "prepublishOnly": "node scripts/prepublish.js"
   },
   "dependencies": {
     "@plutolang/base": "workspace:^",
@@ -34,12 +33,14 @@
     "@types/jest": "^29.5.12",
     "@types/node": "^20",
     "@types/shell-quote": "^1.7.5",
+    "copy-webpack-plugin": "^12.0.2",
     "jest": "^29.7.0",
     "jest-junit": "^16.0.0",
+    "raw-loader": "^4.0.2",
     "ts-jest": "^29.1.2",
-    "typescript": "^5.2.2"
-  },
-  "bundleDependencies": [
-    "pyright-internal"
-  ]
+    "ts-loader": "^9.5.1",
+    "typescript": "^5.2.2",
+    "webpack": "^5.91.0",
+    "webpack-cli": "^5.1.4"
+  }
 }

components/deducers/python-pyright/scripts/prepare-pyright-pkg.sh

@@ -1,4 +1,6 @@
 if [ ! -d "libs/pyright-internal" ]; then
     tar -xvf libs/pyright-internal-1.1.352.tgz -C libs/
     mv libs/package libs/pyright-internal
+    cd libs/pyright-internal
+    npm install --production
 fi

components/deducers/python-pyright/scripts/prepublish.js

@@ -0,0 +1,12 @@
+// Remove dependencies from package.json before publishing.
+// This is because the dependencies are already bundled in the webpack output
+
+const fs = require("fs");
+const path = require("path");
+
+const packagePath = path.join(__dirname, "..", "package.json");
+const packageJson = require(packagePath);
+
+delete packageJson.dependencies;
+
+fs.writeFileSync(packagePath, JSON.stringify(packageJson, null, 2));

components/deducers/python-pyright/src/custom.d.ts

@@ -0,0 +1,4 @@
+declare module '*.txt' {
+  const content: string;
+  export default content;
+}

components/deducers/python-pyright/src/import-finder.ts

@@ -14,23 +14,27 @@ import { ImportResult, ImportType } from "pyright-internal/dist/analyzer/importR
 import { PlatformType } from "@plutolang/base";
 import { Module } from "./module-bundler";
 
-const awsLambdaContainedModulesTxt = {
-  "3.10": path.join(__dirname, "all_aws_lambda_modules_python3.10.txt"),
-};
+import allAwsLambdaModulesPython310 from "./all_aws_lambda_modules_python3.10.txt";
 
-export class ImportFinder {
+function extractRawAwsModules(raw: string) {
   // List of modules that are already included in AWS Lambda environment and should be ignored when
   // packaging the code for AWS Lambda.
-  private static readonly awsLambdaContainedModules: string[] = fs
-    .readFileSync(awsLambdaContainedModulesTxt["3.10"], "utf-8")
+  return raw
     .split("\n")
     .filter((line) => !line.startsWith("#"))
     .map((line) => line.replace(/\.__init__$/g, ""));
+}
+
+export class ImportFinder {
+  private static readonly awsLambdaContainedModules: Map<string, string[]> = new Map([
+    ["python3.10", extractRawAwsModules(allAwsLambdaModulesPython310)],
+  ]);
 
   constructor(
     private readonly importResolver: ImportResolver,
     private readonly execEnv: ExecutionEnvironment,
-    private readonly platform?: PlatformType
+    private readonly platform?: PlatformType,
+    private readonly runtime: string = "python3.10"
   ) {}
 
   public getImportedModulesForSingleFile(sourceFilepath: string): Module[] {
@@ -74,9 +78,10 @@ export class ImportFinder {
 
   private shouldIgnoreForAWS(importInfo: ImportResult) {
     const moduleName = importInfo.importName;
+    const containedModules = ImportFinder.awsLambdaContainedModules.get(this.runtime);
     return (
-      ImportFinder.awsLambdaContainedModules.includes(moduleName) ||
-      ImportFinder.awsLambdaContainedModules.includes(moduleName + ".__init__")
+      containedModules &&
+      (containedModules.includes(moduleName) || containedModules.includes(moduleName + ".__init__"))
     );
   }
 

components/deducers/python-pyright/src/index.ts

@@ -26,12 +26,13 @@ import { ResourceObjectTracker } from "./resource-object-tracker";
 import { CodeSegment, CodeExtractor } from "./code-extractor";
 import { ImportFinder } from "./import-finder";
 import { bundleModules } from "./module-bundler";
+import packageJson from "../package.json";
 
 export default class PyrightDeducer extends core.Deducer {
   //eslint-disable-next-line @typescript-eslint/no-var-requires
-  public readonly name = require(path.join(__dirname, "../package.json")).name;
+  public readonly name = packageJson.name;
   //eslint-disable-next-line @typescript-eslint/no-var-requires
-  public readonly version = require(path.join(__dirname, "../package.json")).version;
+  public readonly version = packageJson.version;
 
   private readonly closureDir: string;
 

components/deducers/python-pyright/src/program-utils.ts

@@ -42,12 +42,24 @@ export function createProgram(options: CreateProgramOptions = {}) {
 }
 
 function resolvePyrightRoot(): string {
-  const modulePath = require.resolve("pyright-internal/dist/analyzer/program");
-  let currentDir = path.dirname(modulePath);
-  while (!fs.existsSync(path.join(currentDir, "package.json"))) {
+  const resolveResult = require.resolve("pyright-internal/dist/analyzer/program");
+  if (typeof resolveResult === "number") {
+    // The result is a number that represents the current process running a production webpack
+    // bundle. The typeshed-fallback directory is copied to the root of the bundle. So we use the
+    // root of the bundle as the the root of pyright-internal.
+    return __dirname;
+  }
+
+  // There are two situations where the result is a string:
+  // 1. In a Jest test environment, the filepath below is an absolute path to the `program.js` file.
+  // 2. In a development webpack environment, the filepath is a relative path to the `program.js`
+  //    file from the root of this package.
+  const filepath = path.resolve(__dirname, "..", resolveResult);
+  let currentDir = path.dirname(filepath);
+  while (!fs.existsSync(path.join(currentDir, "typeshed-fallback"))) {
     const parentDir = path.dirname(currentDir);
     if (parentDir === currentDir) {
-      throw new Error("Cannot find pyright root directory.");
+      throw new Error("Cannot find the root directory of typeshed-fallback.");
     }
     currentDir = parentDir;
   }

components/deducers/python-pyright/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "strict": true,
-    "outDir": "dist",
+    "outDir": "out",
     "target": "es2022",
     "module": "commonjs",
     "lib": ["es2022"],

components/deducers/python-pyright/webpack.config.js

@@ -0,0 +1,34 @@
+const path = require("path");
+const CopyPlugin = require("copy-webpack-plugin");
+
+const outPath = path.resolve(__dirname, "dist");
+const typeshedFallback = path.resolve(__dirname, "libs", "pyright-internal", "typeshed-fallback");
+
+module.exports = {
+  entry: "./src/index.ts",
+  target: "node",
+  output: {
+    path: outPath,
+    filename: "bundle.js",
+    library: {
+      type: "umd",
+    },
+    clean: true,
+  },
+  resolve: {
+    extensions: [".ts", ".tsx", ".js", ".json"],
+  },
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: "ts-loader",
+      },
+      {
+        test: /\.txt$/,
+        use: "raw-loader",
+      },
+    ],
+  },
+  plugins: [new CopyPlugin({ patterns: [{ from: typeshedFallback, to: "typeshed-fallback" }] })],
+};