Merge pull request #1 from mixedbread-ai/feature/mixedbread-ai-integr…

…ation

feat: add mixedbread ai integration

libs/langchain-mixedbreadai/.eslintrc.cjs

@@ -0,0 +1,75 @@
+module.exports = {
+  extends: [
+    "airbnb-base",
+    "eslint:recommended",
+    "prettier",
+    "plugin:@typescript-eslint/recommended",
+  ],
+  parserOptions: {
+    ecmaVersion: 12,
+    parser: "@typescript-eslint/parser",
+    project: "./tsconfig.json",
+    sourceType: "module",
+  },
+  plugins: ["@typescript-eslint", "no-instanceof", "eslint-plugin-jest"],
+  ignorePatterns: [
+    ".eslintrc.cjs",
+    "scripts",
+    "node_modules",
+    "dist",
+    "dist-cjs",
+    "*.js",
+    "*.cjs",
+    "*.d.ts",
+  ],
+  rules: {
+    "no-process-env": 2,
+    "no-instanceof/no-instanceof": 2,
+    "@typescript-eslint/explicit-module-boundary-types": 0,
+    "@typescript-eslint/no-empty-function": 0,
+    "@typescript-eslint/no-shadow": 0,
+    "@typescript-eslint/no-empty-interface": 0,
+    "@typescript-eslint/no-use-before-define": ["error", "nofunc"],
+    "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }],
+    "@typescript-eslint/no-floating-promises": "error",
+    "@typescript-eslint/no-misused-promises": "error",
+    camelcase: 0,
+    "class-methods-use-this": 0,
+    "import/extensions": [2, "ignorePackages"],
+    "import/no-extraneous-dependencies": [
+      "error",
+      { devDependencies: ["**/*.test.ts"] },
+    ],
+    "import/no-unresolved": 0,
+    "import/prefer-default-export": 0,
+    "keyword-spacing": "error",
+    "max-classes-per-file": 0,
+    "max-len": 0,
+    "no-await-in-loop": 0,
+    "no-bitwise": 0,
+    "no-console": 0,
+    "no-restricted-syntax": 0,
+    "no-shadow": 0,
+    "no-continue": 0,
+    "no-void": 0,
+    "no-underscore-dangle": 0,
+    "no-use-before-define": 0,
+    "no-useless-constructor": 0,
+    "no-return-await": 0,
+    "consistent-return": 0,
+    "no-else-return": 0,
+    "func-names": 0,
+    "no-lonely-if": 0,
+    "prefer-rest-params": 0,
+    'jest/no-focused-tests': 'error',
+    "new-cap": ["error", { properties: false, capIsNew: false }],
+  },
+  overrides: [
+    {
+      files: ['**/*.test.ts'],
+      rules: {
+        '@typescript-eslint/no-unused-vars': 'off'
+      }
+    }
+  ]
+};

libs/langchain-mixedbreadai/.gitignore

@@ -0,0 +1,7 @@
+index.cjs
+index.js
+index.d.ts
+index.d.cts
+node_modules
+dist
+.yarn

libs/langchain-mixedbreadai/.prettierrc

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://json.schemastore.org/prettierrc",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "useTabs": false,
+  "semi": true,
+  "singleQuote": false,
+  "quoteProps": "as-needed",
+  "jsxSingleQuote": false,
+  "trailingComma": "es5",
+  "bracketSpacing": true,
+  "arrowParens": "always",
+  "requirePragma": false,
+  "insertPragma": false,
+  "proseWrap": "preserve",
+  "htmlWhitespaceSensitivity": "css",
+  "vueIndentScriptAndStyle": false,
+  "endOfLine": "lf"
+}

libs/langchain-mixedbreadai/.release-it.json

@@ -0,0 +1,12 @@
+{
+  "github": {
+    "release": true,
+    "autoGenerate": true,
+    "tokenRef": "GITHUB_TOKEN_RELEASE"
+  },
+  "npm": {
+    "versionArgs": [
+      "--workspaces-update=false"
+    ]
+  }
+}

libs/langchain-mixedbreadai/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2023 LangChain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

libs/langchain-mixedbreadai/README.md

@@ -0,0 +1,96 @@
+# @langchain/mixedbread-ai
+
+This package contains the LangChain.js integrations for the [mixedbread ai API](https://mixedbread.ai/).
+
+## Installation
+
+```bash npm2yarn
+npm install @langchain/mixedbread-ai
+```
+
+This package, along with the main LangChain package, depends on [`@langchain/core`](https://npmjs.com/package/@langchain/core/).
+If you are using this package with other LangChain packages, you should make sure that all of the packages depend on the same instance of @langchain/core.
+
+## Embeddings
+
+This package provides access to the different embedding models provided by the mixedbread ai API, such as the "mixedbread-ai/mxbai-embed-large-v1" model.
+
+Learn more: [Embeddings API](https://mixedbread.ai/docs/embeddings)
+
+```typescript
+import { MixedbreadAIEmbeddings } from "@langchain/mixedbread-ai";
+
+const embeddings = new MixedbreadAIEmbeddings({
+  apiKey: "YOUR_API_KEY",
+});
+const res = await embeddings.embedQuery("Bread is delicious.");
+console.log(res);
+```
+
+## Reranking
+
+This package provides access to the reranking API provided by the mixedbread ai. It allows you to rerank a list of documents based on a query. Available models include "mixedbread-ai/mxbai-rerank-large-v1".
+
+Learn more: [Reranking API](https://mixedbread.ai/docs/reranking)
+
+```typescript
+import { MixedbreadAIRerank } from "@langchain/mixedbread-ai";
+
+const reranker = new MixedbreadAIRerank({ apiKey: 'your-api-key' });
+const documents = [{ pageContent: "Document 1" }, { pageContent: "Document 2" }];
+const query = "relevant query";
+const res = await reranker.compressDocuments(documents, query);
+console.log(res);
+```
+
+
+## Development
+
+To develop the `@langchain/mixedbread-ai` package, you'll need to follow these instructions:
+
+### Install dependencies
+
+```bash
+yarn install
+```
+
+### Build the package
+
+```bash
+yarn build
+```
+
+Or from the repo root:
+
+```bash
+yarn build --filter=@langchain/mixedbread-ai
+```
+
+### Run tests
+
+Test files should live within a `tests/` file in the `src/` folder. Unit tests should end in `.test.ts` and integration tests should
+end in `.int.test.ts`:
+
+```bash
+$ yarn test
+$ yarn test:int
+```
+
+### Lint & Format
+
+Run the linter & formatter to ensure your code is up to standard:
+
+```bash
+yarn lint && yarn format
+```
+
+### Adding new entrypoints
+
+If you add a new file to be exported, either import & re-export from `src/index.ts`, or add it to the `entrypoints` field in the `config` variable located inside `langchain.config.js` and run `yarn build` to generate the new entrypoint.
+```
+
+### Key Adjustments:
+1. **Package Name**: Changed all instances of `cohere` to `mixedbread-ai`.
+2. **Environment Variable**: Updated the environment variable to `MXBAI_API_KEY`.
+3. **Class Names**: Updated class names to `ChatMixedbreadAI` and `MixedbreadAIEmbeddings`.
+4. **Description**: Adjusted the description to be about MixedbreadAI with a slightly different description.

libs/langchain-mixedbreadai/jest.config.cjs

@@ -0,0 +1,21 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest/presets/default-esm",
+  testEnvironment: "./jest.env.cjs",
+  modulePathIgnorePatterns: ["dist/", "docs/"],
+  moduleNameMapper: {
+    "^(\\.{1,2}/.*)\\.js$": "$1",
+  },
+  transform: {
+    "^.+\\.tsx?$": ["@swc/jest"],
+  },
+  transformIgnorePatterns: [
+    "/node_modules/",
+    "\\.pnp\\.[^\\/]+$",
+    "./scripts/jest-setup-after-env.js",
+  ],
+  setupFiles: ["dotenv/config"],
+  testTimeout: 20_000,
+  passWithNoTests: true,
+  collectCoverageFrom: ["src/**/*.ts"],
+};

libs/langchain-mixedbreadai/jest.env.cjs

@@ -0,0 +1,12 @@
+const { TestEnvironment } = require("jest-environment-node");
+
+class AdjustedTestEnvironmentToSupportFloat32Array extends TestEnvironment {
+  constructor(config, context) {
+    // Make `instanceof Float32Array` return true in tests
+    // to avoid https://github.com/xenova/transformers.js/issues/57 and https://github.com/jestjs/jest/issues/2549
+    super(config, context);
+    this.global.Float32Array = Float32Array;
+  }
+}
+
+module.exports = AdjustedTestEnvironmentToSupportFloat32Array;

libs/langchain-mixedbreadai/langchain.config.js

@@ -0,0 +1,22 @@
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * @param {string} relativePath
+ * @returns {string}
+ */
+function abs(relativePath) {
+  return resolve(dirname(fileURLToPath(import.meta.url)), relativePath);
+}
+
+
+export const config = {
+  internals: [/node\:/, /@langchain\/core\//],
+  entrypoints: {
+    index: "index",
+  },
+  tsConfigPath: resolve("./tsconfig.json"),
+  cjsSource: "./dist-cjs",
+  cjsDestination: "./dist",
+  abs,
+}

libs/langchain-mixedbreadai/package.json

@@ -0,0 +1,89 @@
+{
+  "name": "@langchain/mixedbreadai",
+  "version": "0.1.0",
+  "description": "mixedbread ai integration for LangChain.js",
+  "type": "module",
+  "engines": {
+    "node": ">=18"
+  },
+  "main": "./index.js",
+  "types": "./index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:langchain-ai/langchainjs.git"
+  },
+  "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-mixedbreadai/",
+  "scripts": {
+    "build": "yarn turbo:command build:internal --filter=@langchain/mixedbread-ai",
+    "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking",
+    "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
+    "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts",
+    "lint": "yarn lint:eslint && yarn lint:dpdm",
+    "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm",
+    "clean": "rm -rf .turbo dist/",
+    "prepack": "yarn build",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%",
+    "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts",
+    "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
+    "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%",
+    "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%",
+    "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%",
+    "test:standard": "yarn test:standard:unit && yarn test:standard:int",
+    "format": "prettier --config .prettierrc --write \"src\"",
+    "format:check": "prettier --config .prettierrc --check \"src\""
+  },
+  "author": "LangChain",
+  "license": "MIT",
+  "dependencies": {
+    "@langchain/core": ">=0.2.5 <0.3.0",
+    "@mixedbread-ai/sdk": "^2.2.3"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "@langchain/scripts": "~0.0.14",
+    "@langchain/standard-tests": "0.0.0",
+    "@swc/core": "^1.3.90",
+    "@swc/jest": "^0.2.29",
+    "@tsconfig/recommended": "^1.0.3",
+    "@typescript-eslint/eslint-plugin": "^6.12.0",
+    "@typescript-eslint/parser": "^6.12.0",
+    "dotenv": "^16.3.1",
+    "dpdm": "^3.12.0",
+    "eslint": "^8.33.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-config-prettier": "^8.6.0",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-jest": "^27.6.0",
+    "eslint-plugin-no-instanceof": "^1.0.1",
+    "eslint-plugin-prettier": "^4.2.1",
+    "jest": "^29.5.0",
+    "jest-environment-node": "^29.6.4",
+    "prettier": "^2.8.3",
+    "release-it": "^15.10.1",
+    "rollup": "^4.5.2",
+    "ts-jest": "^29.1.0",
+    "typescript": "<5.2.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": {
+        "import": "./index.d.ts",
+        "require": "./index.d.cts",
+        "default": "./index.d.ts"
+      },
+      "import": "./index.js",
+      "require": "./index.cjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist/",
+    "index.cjs",
+    "index.js",
+    "index.d.ts",
+    "index.d.cts"
+  ]
+}

libs/langchain-mixedbreadai/scripts/jest-setup-after-env.js

@@ -0,0 +1,3 @@
+import { awaitAllCallbacks } from "@langchain/core/callbacks/promises";
+
+afterAll(awaitAllCallbacks);