diff --git a/docs/core_docs/docs/guides/migrating.mdx b/docs/core_docs/docs/guides/migrating.mdx new file mode 100644 index 000000000000..4fa95fec16f9 --- /dev/null +++ b/docs/core_docs/docs/guides/migrating.mdx @@ -0,0 +1,53 @@ +--- +title: Migrating to 0.1 +hide_table_of_contents: true +--- + +# Migration guide: 0.0 -> 0.1 + +If you're still using the pre `0.1` version of LangChain, but want to upgrade to the latest version, we've created a script that can handle almost every aspect of the migration for you. + +At a high level, the changes from `0.0` to `0.1` are new packages and import path updates. +We've done our best to keep all core code functionality the same, so migrating can be as painless as possible. + +In simple terms, this script will scan your TypeScript codebase for any imports from `langchain/*`, and if it finds imports which have been moved in `0.1`, it'll automatically update the import paths for you. + +The new packages it checks for are: + +- `@langchain/core` +- `@langchain/community` +- `@langchain/openai` +- `@langchain/cohere` +- `@langchain/pinecone` + +Some of these integration packages (not `core` or `community`) do have breaking changes. If you'd like to opt out of updating to those modules, you may pass in the `skipCheck` arg with a list of modules you'd like to ignore. + +For example, `@langchain/cohere` bumps to the new Cohere SDK version. If you do not wish to upgrade, it will instead update your `cohere` imports to the `@langchain/community` package which still contains the previous version of the Cohere SDK. + +The example below demonstrates how to run the migration script, checking all new packages. + +### Setup + +Install the new packages. + +import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx"; +import CodeBlock from "@theme/CodeBlock"; +import MigrationExample from "@examples/guides/migrating.ts"; + + + +Install the scripts package to import the migration script: + +```bash npm2yarn +npm install @langchain/scripts +``` + +Then, install any integration packages you'd like to use: + +```bash npm2yarn +npm install @langchain/core @langchain/community @langchain/openai @langchain/cohere @langchain/pinecone +``` + +Then, run the migration code as seen below. + +{MigrationExample} diff --git a/examples/package.json b/examples/package.json index 228f0fd65e2f..ee87efd27620 100644 --- a/examples/package.json +++ b/examples/package.json @@ -41,6 +41,7 @@ "@langchain/openai": "workspace:*", "@langchain/pinecone": "workspace:*", "@langchain/redis": "workspace:*", + "@langchain/scripts": "workspace:*", "@langchain/weaviate": "workspace:*", "@langchain/yandex": "workspace:*", "@opensearch-project/opensearch": "^2.2.0", diff --git a/examples/src/guides/migrating.ts b/examples/src/guides/migrating.ts new file mode 100644 index 000000000000..e78ae9757daf --- /dev/null +++ b/examples/src/guides/migrating.ts @@ -0,0 +1,8 @@ +import { updateEntrypointsFrom0_0_xTo0_1_x } from "@langchain/scripts/migrations"; + +await updateEntrypointsFrom0_0_xTo0_1_x({ + // Path to the local langchainjs repository + localLangChainPath: "/Users/my-profile/langchainjs", + // Path to the repository where the migration should be applied + codePath: "/Users/my-profile/langchainjs-project", +}); diff --git a/libs/langchain-scripts/.gitignore b/libs/langchain-scripts/.gitignore index 9f044b48439f..0aadd3b51862 100644 --- a/libs/langchain-scripts/.gitignore +++ b/libs/langchain-scripts/.gitignore @@ -4,6 +4,9 @@ index.d.ts build.cjs build.js build.d.ts +migrations.cjs +migrations.js +migrations.d.ts node_modules dist .yarn diff --git a/libs/langchain-scripts/package.json b/libs/langchain-scripts/package.json index ace040cecabc..6151f96e3f47 100644 --- a/libs/langchain-scripts/package.json +++ b/libs/langchain-scripts/package.json @@ -38,7 +38,9 @@ "license": "MIT", "dependencies": { "commander": "^11.1.0", + "glob": "^10.3.10", "rollup": "^4.5.2", + "ts-morph": "^21.0.1", "typescript": "<5.2.0" }, "devDependencies": { @@ -76,6 +78,11 @@ "import": "./build.js", "require": "./build.cjs" }, + "./migrations": { + "types": "./migrations.d.ts", + "import": "./migrations.js", + "require": "./migrations.cjs" + }, "./package.json": "./package.json" }, "files": [ @@ -85,6 +92,9 @@ "index.d.ts", "build.cjs", "build.js", - "build.d.ts" + "build.d.ts", + "migrations.cjs", + "migrations.js", + "migrations.d.ts" ] } diff --git a/libs/langchain-scripts/scripts/create-entrypoints.js b/libs/langchain-scripts/scripts/create-entrypoints.js index a5e34c5120ba..7197390a066d 100644 --- a/libs/langchain-scripts/scripts/create-entrypoints.js +++ b/libs/langchain-scripts/scripts/create-entrypoints.js @@ -12,6 +12,7 @@ const DEFAULT_GITIGNORE_PATHS = ["node_modules", "dist", ".yarn"]; const entrypoints = { index: "index", build: "build", + migrations: "migrations/index", }; // Entrypoints in this list require an optional dependency to be installed. diff --git a/libs/langchain-scripts/src/migrations/0_1.ts b/libs/langchain-scripts/src/migrations/0_1.ts new file mode 100644 index 000000000000..84ab7c662d37 --- /dev/null +++ b/libs/langchain-scripts/src/migrations/0_1.ts @@ -0,0 +1,373 @@ +import { ImportSpecifier, Project, SourceFile, SyntaxKind } from "ts-morph"; +import { glob } from "glob"; +import path from "node:path"; +import { LangChainConfig } from "../types.js"; + +type ExportedSymbol = { symbol: string; kind: SyntaxKind }; + +type EntrypointAndSymbols = { + entrypoint: string; + exportedSymbols: Array; +}; + +const enum UpgradingModule { + COHERE = "cohere", + PINECONE = "pinecone", +} + +/** + * @param {string} packagePath + * @param {Project} project + * @returns {Array } + */ +async function getEntrypointsFromFile( + packagePath: string, + project: Project +): Promise> { + const { config }: { config: LangChainConfig } = await import( + path.join(packagePath, "langchain.config.js") + ); + const { entrypoints, deprecatedNodeOnly } = config; + + const result = Object.entries(entrypoints).flatMap(([key, value]) => { + if (deprecatedNodeOnly?.includes(key)) { + return []; + } + const newFile = project.addSourceFileAtPath( + path.join(packagePath, "src", `${value}.ts`) + ); + const exportedSymbolsMap = newFile.getExportedDeclarations(); + const exportedSymbols = Array.from(exportedSymbolsMap.entries()).map( + ([symbol, declarations]) => ({ + kind: declarations[0].getKind(), + symbol, + }) + ); + return { + entrypoint: key, + exportedSymbols, + }; + }); + + return result; +} + +type FoundSymbol = { + entrypoint: string; + foundSymbol: string; + packageSuffix: string; +}; + +/** + * Finds a matching symbol in the array of exported symbols. + * @param {{ symbol: string, kind: SyntaxKind }} target - The target symbol and its kind to find. + * @param {Array} exportedSymbols - The array of exported symbols to search. + * @param {string} packageSuffix - The suffix of the package to import from. Eg, core + * @returns {{ entrypoint: string, foundSymbol: string } | undefined} The matching symbol or undefined if not found. + */ +function findMatchingSymbol( + target: { symbol: string; kind: SyntaxKind }, + exportedSymbols: Array, + packageSuffix: string +): FoundSymbol | undefined { + for (const entry of exportedSymbols) { + const foundSymbol = entry.exportedSymbols.find( + ({ symbol, kind }) => symbol === target.symbol && kind === target.kind + ); + if (foundSymbol) { + return { + entrypoint: entry.entrypoint, + foundSymbol: foundSymbol.symbol, + packageSuffix, + }; // Return the matching entry object + } + } + return undefined; +} + +/** + * @param {Array} entrypoints + * @returns {Array} + */ +function removeLoad( + entrypoints: Array +): Array { + return entrypoints.flatMap((entrypoint) => { + const newEntrypoint = + entrypoint.entrypoint === "index" ? "" : `/${entrypoint.entrypoint}`; + const withoutLoadOrIndex = entrypoint.exportedSymbols.filter((item) => { + if (item.symbol === "load" && newEntrypoint === "load") { + return false; + } + return true; + }); + return { + entrypoint: newEntrypoint, + exportedSymbols: withoutLoadOrIndex, + }; + }); +} + +function updateImport({ + matchingSymbols, + namedImport, + projectFile, + namedImportText, +}: { + matchingSymbols: Array; + namedImport: ImportSpecifier; + projectFile: SourceFile; + namedImportText: string; +}): boolean { + const firstMatchingSymbol = matchingSymbols.find( + (matchingSymbol) => matchingSymbol + ); + if (firstMatchingSymbol) { + console.debug( + `Found matching symbol in the "@langchain/${firstMatchingSymbol.packageSuffix}" package.`, + { + matchingSymbol: firstMatchingSymbol, + } + ); + + namedImport.remove(); + projectFile.addImportDeclaration({ + moduleSpecifier: `@langchain/${firstMatchingSymbol.packageSuffix}${firstMatchingSymbol.entrypoint}`, + namedImports: [namedImportText], + }); + return true; + } + return false; +} + +/** + * Find imports from deprecated pre 0.1 LangChain modules and update them to import + * from the new LangChain packages. + */ +export async function updateEntrypointsFrom0_0_xTo0_1_x({ + localLangChainPath, + codePath, + customGlobPattern, + customIgnorePattern, + skipCheck, +}: { + /** + * The absolute path to the locally cloned LangChain repo root. + * @example "/Users/username/code/langchainjs" + */ + localLangChainPath: string; + /** + * The absolute path to the source directory of the codebase to update. + * @example "/Users/username/code/my-project/src" + */ + codePath: string; + /** + * Optionally, pass in a custom glob pattern to match files. + * The backslash included in the example and default is only for + * JSDoc to escape the asterisk. Do not include unless intentionally. + * @example "/*.d.ts" + * @default "**\/*.ts" + */ + customGlobPattern?: string; + /** + * A custom ignore pattern for ignoring files. + * The backslash included in the example and default is only for + * JSDoc to escape the asterisk. Do not include unless intentionally. + * @example ["**\/node_modules/**", "**\/dist/**", "**\/*.d.ts"] + * @default node_modules/** + */ + customIgnorePattern?: string[] | string; + /** + * Optionally skip checking the passed modules for imports to + * update. + * @example [UpgradingModule.COHERE] + * @default undefined + */ + skipCheck?: Array; +}) { + const project = new Project(); + + const langchainCorePackageEntrypoints = removeLoad( + await getEntrypointsFromFile( + path.join(localLangChainPath, "langchain-core"), + project + ) + ); + const langchainCommunityPackageEntrypoints = removeLoad( + await getEntrypointsFromFile( + path.join(localLangChainPath, "libs", "langchain-community"), + project + ) + ); + const langchainOpenAIPackageEntrypoints = removeLoad( + await getEntrypointsFromFile( + path.join(localLangChainPath, "libs", "langchain-openai"), + project + ) + ); + const langchainCoherePackageEntrypoints = !skipCheck?.includes( + UpgradingModule.COHERE + ) + ? removeLoad( + await getEntrypointsFromFile( + path.join(localLangChainPath, "libs", "langchain-cohere"), + project + ) + ) + : null; + const langchainPineconePackageEntrypoints = !skipCheck?.includes( + UpgradingModule.PINECONE + ) + ? removeLoad( + await getEntrypointsFromFile( + path.join(localLangChainPath, "libs", "langchain-pinecone"), + project + ) + ) + : null; + + const globPattern = customGlobPattern || "/**/*.ts"; + const ignorePattern = customIgnorePattern; + + const allCodebaseFiles = ( + await glob(path.join(codePath, globPattern), { + ignore: ignorePattern, + }) + ) + .map((filePath) => path.resolve(filePath)) + .filter((filePath) => !filePath.includes("node_modules/")); + + for await (const filePath of allCodebaseFiles) { + let projectFile: SourceFile; + try { + projectFile = project.addSourceFileAtPath(filePath); + if (!projectFile) { + throw new Error(`Failed to add source file at path: ${filePath}`); + } + } catch (error) { + console.error( + { + filePath, + error, + }, + "Error occurred while trying to add source file. Continuing" + ); + return; + } + + try { + const imports = projectFile.getImportDeclarations(); + + imports.forEach((importItem) => { + // Get all imports + const module = importItem.getModuleSpecifierValue(); + // Get only the named imports. Eg: import { foo } from "langchain/util"; + const namedImports = importItem.getNamedImports(); + if (!module.startsWith("langchain/")) { + return; + } + + // look at each import and see if it exists in + let didUpdate = false; + + namedImports.forEach((namedImport) => { + const namedImportText = namedImport.getText(); + let namedImportKind: SyntaxKind | null = null; + + const symbol = namedImport.getSymbol(); + if (symbol) { + // Resolve alias symbol to its original symbol + const aliasedSymbol = symbol.getAliasedSymbol() || symbol; + + // Get the original declarations of the symbol + const declarations = aliasedSymbol.getDeclarations(); + if (declarations.length > 0) { + // Assuming the first declaration is the original one + const originalDeclarationKind = declarations[0].getKind(); + namedImportKind = originalDeclarationKind; + } + } + + // If we couldn't find the kind of the named imports kind, skip it + if (!namedImportKind) { + return; + } + + const matchingSymbolCore = findMatchingSymbol( + { symbol: namedImportText, kind: namedImportKind }, + langchainCorePackageEntrypoints, + "core" + ); + const matchingSymbolCommunity = findMatchingSymbol( + { symbol: namedImportText, kind: namedImportKind }, + langchainCommunityPackageEntrypoints, + "community" + ); + const matchingSymbolOpenAI = findMatchingSymbol( + { symbol: namedImportText, kind: namedImportKind }, + langchainOpenAIPackageEntrypoints, + "openai" + ); + const matchingSymbolCohere = langchainCoherePackageEntrypoints + ? findMatchingSymbol( + { symbol: namedImportText, kind: namedImportKind }, + langchainCoherePackageEntrypoints, + "cohere" + ) + : undefined; + const matchingSymbolPinecone = langchainPineconePackageEntrypoints + ? findMatchingSymbol( + { symbol: namedImportText, kind: namedImportKind }, + langchainPineconePackageEntrypoints, + "pinecone" + ) + : undefined; + + didUpdate = updateImport({ + matchingSymbols: [ + matchingSymbolCore, + matchingSymbolOpenAI, + matchingSymbolCohere, + matchingSymbolPinecone, + matchingSymbolCommunity, + ], + namedImport, + projectFile, + namedImportText, + }); + }); + + if (didUpdate) { + projectFile.saveSync(); + + // Check if all named imports were removed, and only a file import remains. + // eg: import { foo } from "langchain/anthropic"; -> import "langchain/anthropic"; + // if so, remove the import entirely + const importClause = importItem.getImportClause(); + if ( + !importClause || + (!importClause.getDefaultImport() && + importClause.getNamedImports().length === 0) + ) { + importItem.remove(); + projectFile.saveSync(); + } + } + }); + } catch (error) { + console.error( + { + filePath, + error, + }, + "Error occurred while trying to read file. Continuing" + ); + } + + // Remove source file from the project after we're done with it + // to prevent OOM errors. + if (projectFile) { + project.removeSourceFile(projectFile); + } + } +} diff --git a/libs/langchain-scripts/src/migrations/index.ts b/libs/langchain-scripts/src/migrations/index.ts new file mode 100644 index 000000000000..f809318e4235 --- /dev/null +++ b/libs/langchain-scripts/src/migrations/index.ts @@ -0,0 +1 @@ +export * from "./0_1.js"; diff --git a/yarn.lock b/yarn.lock index 120f97f79b12..7c892f5ede43 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9556,7 +9556,7 @@ __metadata: languageName: unknown linkType: soft -"@langchain/scripts@workspace:libs/langchain-scripts, @langchain/scripts@~0.0": +"@langchain/scripts@workspace:*, @langchain/scripts@workspace:libs/langchain-scripts, @langchain/scripts@~0.0": version: 0.0.0-use.local resolution: "@langchain/scripts@workspace:libs/langchain-scripts" dependencies: @@ -9575,12 +9575,14 @@ __metadata: eslint-plugin-import: ^2.27.5 eslint-plugin-no-instanceof: ^1.0.1 eslint-plugin-prettier: ^4.2.1 + glob: ^10.3.10 jest: ^29.5.0 jest-environment-node: ^29.6.4 prettier: ^2.8.3 release-it: ^15.10.1 rollup: ^4.5.2 ts-jest: ^29.1.0 + ts-morph: ^21.0.1 typescript: <5.2.0 bin: lc-build: ./build.js @@ -13588,6 +13590,18 @@ __metadata: languageName: node linkType: hard +"@ts-morph/common@npm:~0.22.0": + version: 0.22.0 + resolution: "@ts-morph/common@npm:0.22.0" + dependencies: + fast-glob: ^3.3.2 + minimatch: ^9.0.3 + mkdirp: ^3.0.1 + path-browserify: ^1.0.1 + checksum: e549facfff2a68eeef4e3e2c4183e7216a02b57e62cdfe60ca15d5fdee24770bd3b5b6d1a0388cfce7b4dfaeb0ebe31ffa40585e36b9fb7948aea8081fa73769 + languageName: node + linkType: hard + "@tsconfig/recommended@npm:^1.0.2": version: 1.0.2 resolution: "@tsconfig/recommended@npm:1.0.2" @@ -17746,6 +17760,13 @@ __metadata: languageName: node linkType: hard +"code-block-writer@npm:^12.0.0": + version: 12.0.0 + resolution: "code-block-writer@npm:12.0.0" + checksum: 9f6505a4d668c9131c6f3f686359079439e66d5f50c236614d52fcfa53aeb0bc615b2c6c64ef05b5511e3b0433ccfd9f7756ad40eb3b9298af6a7d791ab1981d + languageName: node + linkType: hard + "cohere-ai@npm:>=6.0.0": version: 6.2.2 resolution: "cohere-ai@npm:6.2.2" @@ -20919,6 +20940,7 @@ __metadata: "@langchain/openai": "workspace:*" "@langchain/pinecone": "workspace:*" "@langchain/redis": "workspace:*" + "@langchain/scripts": "workspace:*" "@langchain/weaviate": "workspace:*" "@langchain/yandex": "workspace:*" "@opensearch-project/opensearch": ^2.2.0 @@ -22312,6 +22334,21 @@ __metadata: languageName: node linkType: hard +"glob@npm:^10.3.10": + version: 10.3.10 + resolution: "glob@npm:10.3.10" + dependencies: + foreground-child: ^3.1.0 + jackspeak: ^2.3.5 + minimatch: ^9.0.1 + minipass: ^5.0.0 || ^6.0.2 || ^7.0.0 + path-scurry: ^1.10.1 + bin: + glob: dist/esm/bin.mjs + checksum: 4f2fe2511e157b5a3f525a54092169a5f92405f24d2aed3142f4411df328baca13059f4182f1db1bf933e2c69c0bd89e57ae87edd8950cba8c7ccbe84f721cf3 + languageName: node + linkType: hard + "glob@npm:^7.0.0, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6": version: 7.2.3 resolution: "glob@npm:7.2.3" @@ -24383,6 +24420,19 @@ __metadata: languageName: node linkType: hard +"jackspeak@npm:^2.3.5": + version: 2.3.6 + resolution: "jackspeak@npm:2.3.6" + dependencies: + "@isaacs/cliui": ^8.0.2 + "@pkgjs/parseargs": ^0.11.0 + dependenciesMeta: + "@pkgjs/parseargs": + optional: true + checksum: 57d43ad11eadc98cdfe7496612f6bbb5255ea69fe51ea431162db302c2a11011642f50cfad57288bd0aea78384a0612b16e131944ad8ecd09d619041c8531b54 + languageName: node + linkType: hard + "jest-changed-files@npm:^29.5.0": version: 29.5.0 resolution: "jest-changed-files@npm:29.5.0" @@ -26395,6 +26445,13 @@ __metadata: languageName: node linkType: hard +"lru-cache@npm:^9.1.1 || ^10.0.0": + version: 10.2.0 + resolution: "lru-cache@npm:10.2.0" + checksum: eee7ddda4a7475deac51ac81d7dd78709095c6fa46e8350dc2d22462559a1faa3b81ed931d5464b13d48cbd7e08b46100b6f768c76833912bc444b99c37e25db + languageName: node + linkType: hard + "lru-cache@npm:^9.1.1, lru-cache@npm:^9.1.2": version: 9.1.2 resolution: "lru-cache@npm:9.1.2" @@ -27031,6 +27088,13 @@ __metadata: languageName: node linkType: hard +"minipass@npm:^5.0.0 || ^6.0.2 || ^7.0.0": + version: 7.0.4 + resolution: "minipass@npm:7.0.4" + checksum: 87585e258b9488caf2e7acea242fd7856bbe9a2c84a7807643513a338d66f368c7d518200ad7b70a508664d408aa000517647b2930c259a8b1f9f0984f344a21 + languageName: node + linkType: hard + "minizlib@npm:^2.0.0, minizlib@npm:^2.1.1, minizlib@npm:^2.1.2": version: 2.1.2 resolution: "minizlib@npm:2.1.2" @@ -27073,6 +27137,15 @@ __metadata: languageName: node linkType: hard +"mkdirp@npm:^3.0.1": + version: 3.0.1 + resolution: "mkdirp@npm:3.0.1" + bin: + mkdirp: dist/cjs/src/bin.js + checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d + languageName: node + linkType: hard + "ml-array-max@npm:^1.2.4": version: 1.2.4 resolution: "ml-array-max@npm:1.2.4" @@ -28633,6 +28706,13 @@ __metadata: languageName: node linkType: hard +"path-browserify@npm:^1.0.1": + version: 1.0.1 + resolution: "path-browserify@npm:1.0.1" + checksum: c6d7fa376423fe35b95b2d67990060c3ee304fc815ff0a2dc1c6c3cfaff2bd0d572ee67e18f19d0ea3bbe32e8add2a05021132ac40509416459fffee35200699 + languageName: node + linkType: hard + "path-exists@npm:^3.0.0": version: 3.0.0 resolution: "path-exists@npm:3.0.0" @@ -28682,6 +28762,16 @@ __metadata: languageName: node linkType: hard +"path-scurry@npm:^1.10.1": + version: 1.10.1 + resolution: "path-scurry@npm:1.10.1" + dependencies: + lru-cache: ^9.1.1 || ^10.0.0 + minipass: ^5.0.0 || ^6.0.2 || ^7.0.0 + checksum: e2557cff3a8fb8bc07afdd6ab163a92587884f9969b05bbbaf6fe7379348bfb09af9ed292af12ed32398b15fb443e81692047b786d1eeb6d898a51eb17ed7d90 + languageName: node + linkType: hard + "path-scurry@npm:^1.7.0": version: 1.9.2 resolution: "path-scurry@npm:1.9.2" @@ -33303,6 +33393,16 @@ __metadata: languageName: node linkType: hard +"ts-morph@npm:^21.0.1": + version: 21.0.1 + resolution: "ts-morph@npm:21.0.1" + dependencies: + "@ts-morph/common": ~0.22.0 + code-block-writer: ^12.0.0 + checksum: f8e6acd4cdb2842af47ccf4e8900dc3f230f20c3b0d28e1e8b58c395b0a16d7b3e03ef56f29da3fdb861c50e22eb52524e0fc4bfca0fde8448f81b8f4f6aa157 + languageName: node + linkType: hard + "ts-type@npm:>=2": version: 3.0.1 resolution: "ts-type@npm:3.0.1"