community[minor],docs[minor]: Add UpstashVector (#4288)

* add upstash vector integration

* bump @upstash/vector

* add docs

* add chunk support, improve docs

* address review

* update vector sdk version

* Apply suggestions from code review

* update example

* format

* fix examples build

* langchain[patch]: Make sitemap test integration (#4358)

* rm from langchain proper

---------

Co-authored-by: Brace Sproul <braceasproul@gmail.com>

docs/core_docs/docs/integrations/vectorstores/upstash.mdx

@@ -0,0 +1,48 @@
+import CodeBlock from "@theme/CodeBlock";
+import CreateClientExample from "@examples/indexes/vector_stores/upstash/create_client.ts";
+import IndexQueryExample from "@examples/indexes/vector_stores/upstash/index_and_query_docs.ts";
+import DeleteExample from "@examples/indexes/vector_stores/upstash/delete_docs.ts";
+import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx";
+
+# Upstash Vector
+
+Upstash Vector is a REST based serverless vector database, designed for working with vector embeddings.
+
+## Setup
+
+1. Create Upstash Vector Index
+
+You can create an index from [Upstash Console](https://console.upstash.com/vector). For further reference, see [docs](https://upstash.com/docs/vector/overall/getstarted).
+
+2. Install Upstash Vector SDK.
+
+```bash npm2yarn
+npm install -S @upstash/vector
+```
+
+We use OpenAI for the embeddings of the below examples.
+However, you can also create the embeddings using the model of your choice, that is available in the LangChain.
+
+<IntegrationInstallTooltip></IntegrationInstallTooltip>
+
+```bash npm2yarn
+npm install @langchain/openai @langchain/community
+```
+
+## Create Upstash Vector Client
+
+There are two ways to create the client. You can either pass the credentials as string manually from the `.env` file (or as string variables), or you can retrieve the credentials from the environment automatically.
+
+<CodeBlock language="typescript">{CreateClientExample}</CodeBlock>
+
+## Index and Query Documents
+
+You can index the LangChain documents with any model of your choice, and perform a search over these documents.
+
+<CodeBlock language="typescript">{IndexQueryExample}</CodeBlock>
+
+## Delete Documents
+
+You can also delete the documents you've indexed previously.
+
+<CodeBlock language="typescript">{DeleteExample}</CodeBlock>

examples/package.json

@@ -53,6 +53,7 @@
     "@supabase/supabase-js": "^2.10.0",
     "@tensorflow/tfjs-backend-cpu": "^4.4.0",
     "@upstash/redis": "^1.20.6",
+    "@upstash/vector": "^1.0.2",
     "@vercel/kv": "^0.2.3",
     "@xata.io/client": "^0.28.0",
     "@zilliz/milvus2-sdk-node": "^2.2.7",

examples/src/indexes/vector_stores/upstash/create_client.ts

@@ -0,0 +1,22 @@
+import { Index } from "@upstash/vector";
+import { OpenAIEmbeddings } from "@langchain/openai";
+import { UpstashVectorStore } from "@langchain/community/vectorstores/upstash";
+
+const embeddings = new OpenAIEmbeddings({});
+
+// Creating the index with the provided credentials.
+const indexWithCredentials = new Index({
+  url: process.env.UPSTASH_VECTOR_REST_URL as string,
+  token: process.env.UPSTASH_VECTOR_REST_TOKEN as string,
+});
+
+const storeWithCredentials = new UpstashVectorStore(embeddings, {
+  index: indexWithCredentials,
+});
+
+// Creating the index from the environment variables automatically.
+const indexFromEnv = new Index();
+
+const storeFromEnv = new UpstashVectorStore(embeddings, {
+  index: indexFromEnv,
+});

examples/src/indexes/vector_stores/upstash/delete_docs.ts

@@ -0,0 +1,28 @@
+import { Index } from "@upstash/vector";
+import { OpenAIEmbeddings } from "@langchain/openai";
+import { UpstashVectorStore } from "@langchain/community/vectorstores/upstash";
+
+const index = new Index({
+  url: process.env.UPSTASH_VECTOR_REST_URL as string,
+  token: process.env.UPSTASH_VECTOR_REST_TOKEN as string,
+});
+
+const embeddings = new OpenAIEmbeddings({});
+
+const UpstashVector = new UpstashVectorStore(embeddings, { index });
+
+// Creating the docs to be indexed.
+const createdAt = new Date().getTime();
+
+const IDs = await UpstashVector.addDocuments([
+  { pageContent: "hello", metadata: { a: createdAt + 1 } },
+  { pageContent: "car", metadata: { a: createdAt } },
+  { pageContent: "adjective", metadata: { a: createdAt } },
+  { pageContent: "hi", metadata: { a: createdAt } },
+]);
+
+// Waiting vectors to be indexed in the vector store.
+// eslint-disable-next-line no-promise-executor-return
+await new Promise((resolve) => setTimeout(resolve, 1000));
+
+await UpstashVector.delete({ ids: [IDs[0], IDs[2], IDs[3]] });

examples/src/indexes/vector_stores/upstash/index_and_query_docs.ts

@@ -0,0 +1,66 @@
+import { Index } from "@upstash/vector";
+import { OpenAIEmbeddings } from "@langchain/openai";
+import { Document } from "@langchain/core/documents";
+import { UpstashVectorStore } from "@langchain/community/vectorstores/upstash";
+
+const index = new Index({
+  url: process.env.UPSTASH_VECTOR_REST_URL as string,
+  token: process.env.UPSTASH_VECTOR_REST_TOKEN as string,
+});
+
+const embeddings = new OpenAIEmbeddings({});
+
+const UpstashVector = new UpstashVectorStore(embeddings, { index });
+
+// Creating the docs to be indexed.
+const id = new Date().getTime();
+const documents = [
+  new Document({
+    metadata: { name: id },
+    pageContent: "Hello there!",
+  }),
+  new Document({
+    metadata: { name: id },
+    pageContent: "What are you building?",
+  }),
+  new Document({
+    metadata: { time: id },
+    pageContent: "Upstash Vector is great for building AI applications.",
+  }),
+  new Document({
+    metadata: { time: id },
+    pageContent: "To be, or not to be, that is the question.",
+  }),
+];
+
+// Creating embeddings from the provided documents, and adding them to Upstash database.
+await UpstashVector.addDocuments(documents);
+
+// Waiting vectors to be indexed in the vector store.
+// eslint-disable-next-line no-promise-executor-return
+await new Promise((resolve) => setTimeout(resolve, 1000));
+
+const queryResult = await UpstashVector.similaritySearchWithScore(
+  "Vector database",
+  2
+);
+
+console.log(queryResult);
+/**
+[
+  [
+    Document {
+      pageContent: 'Upstash Vector is great for building AI applications.',
+      metadata: [Object]
+    },
+    0.9016147
+  ],
+  [
+    Document {
+      pageContent: 'What are you building?',
+      metadata: [Object]
+    },
+    0.8613077
+  ]
+]
+ */

langchain/langchain.config.js

@@ -9,7 +9,6 @@ function abs(relativePath) {
   return resolve(dirname(fileURLToPath(import.meta.url)), relativePath);
 }
 
-
 export const config = {
   internals: [
     /node\:/,
@@ -858,4 +857,4 @@ export const config = {
   cjsSource: "./dist-cjs",
   cjsDestination: "./dist",
   abs,
-}
+};

libs/langchain-community/.gitignore

@@ -386,6 +386,10 @@ vectorstores/typesense.cjs
 vectorstores/typesense.js
 vectorstores/typesense.d.ts
 vectorstores/typesense.d.cts
+vectorstores/upstash.cjs
+vectorstores/upstash.js
+vectorstores/upstash.d.ts
+vectorstores/upstash.d.cts
 vectorstores/usearch.cjs
 vectorstores/usearch.js
 vectorstores/usearch.d.ts

libs/langchain-community/langchain.config.js

@@ -127,6 +127,7 @@ export const config = {
     "vectorstores/turbopuffer": "vectorstores/turbopuffer",
     "vectorstores/typeorm": "vectorstores/typeorm",
     "vectorstores/typesense": "vectorstores/typesense",
+    "vectorstores/upstash": "vectorstores/upstash",
     "vectorstores/usearch": "vectorstores/usearch",
     "vectorstores/vectara": "vectorstores/vectara",
     "vectorstores/vercel_postgres": "vectorstores/vercel_postgres",
@@ -282,6 +283,7 @@ export const config = {
     "vectorstores/tigris",
     "vectorstores/typeorm",
     "vectorstores/typesense",
+    "vectorstores/upstash",
     "vectorstores/usearch",
     "vectorstores/vercel_postgres",
     "vectorstores/voy",

libs/langchain-community/package.json

@@ -106,6 +106,7 @@
     "@typescript-eslint/eslint-plugin": "^5.58.0",
     "@typescript-eslint/parser": "^5.58.0",
     "@upstash/redis": "^1.20.6",
+    "@upstash/vector": "^1.0.2",
     "@vercel/kv": "^0.2.3",
     "@vercel/postgres": "^0.5.0",
     "@writerai/writer-sdk": "^0.40.2",
@@ -207,6 +208,7 @@
     "@tensorflow/tfjs-converter": "*",
     "@tensorflow/tfjs-core": "*",
     "@upstash/redis": "^1.20.6",
+    "@upstash/vector": "^1.0.2",
     "@vercel/kv": "^0.2.3",
     "@vercel/postgres": "^0.5.0",
     "@writerai/writer-sdk": "^0.40.2",
@@ -368,6 +370,9 @@
     "@upstash/redis": {
       "optional": true
     },
+    "@upstash/vector": {
+      "optional": true
+    },
     "@vercel/kv": {
       "optional": true
     },
@@ -1381,6 +1386,15 @@
       "import": "./vectorstores/typesense.js",
       "require": "./vectorstores/typesense.cjs"
     },
+    "./vectorstores/upstash": {
+      "types": {
+        "import": "./vectorstores/upstash.d.ts",
+        "require": "./vectorstores/upstash.d.cts",
+        "default": "./vectorstores/upstash.d.ts"
+      },
+      "import": "./vectorstores/upstash.js",
+      "require": "./vectorstores/upstash.cjs"
+    },
     "./vectorstores/usearch": {
       "types": {
         "import": "./vectorstores/usearch.d.ts",
@@ -2448,6 +2462,10 @@
     "vectorstores/typesense.js",
     "vectorstores/typesense.d.ts",
     "vectorstores/typesense.d.cts",
+    "vectorstores/upstash.cjs",
+    "vectorstores/upstash.js",
+    "vectorstores/upstash.d.ts",
+    "vectorstores/upstash.d.cts",
     "vectorstores/usearch.cjs",
     "vectorstores/usearch.js",
     "vectorstores/usearch.d.ts",

libs/langchain-community/src/vectorstores/tests/upstash.int.test.ts

@@ -0,0 +1,86 @@
+/* eslint-disable no-process-env */
+import { Index } from "@upstash/vector";
+import { Document } from "@langchain/core/documents";
+import { SyntheticEmbeddings } from "@langchain/core/utils/testing";
+import { EmbeddingsInterface } from "@langchain/core/embeddings";
+import { UpstashVectorStore } from "../upstash.js";
+import { sleep } from "../../utils/time.js";
+
+describe.skip("UpstashVectorStore", () => {
+  let store: UpstashVectorStore;
+  let embeddings: EmbeddingsInterface;
+  let index: Index;
+
+  beforeEach(async () => {
+    index = new Index({
+      url: process.env.UPSTASH_VECTOR_REST_URL,
+      token: process.env.UPSTASH_VECTOR_REST_TOKEN,
+    });
+
+    embeddings = new SyntheticEmbeddings({
+      vectorSize: 1536,
+    });
+
+    store = new UpstashVectorStore(embeddings, {
+      index,
+    });
+
+    expect(store).toBeDefined();
+  });
+
+  test("basic operations with documents", async () => {
+    const createdAt = new Date().getTime();
+
+    const ids = await store.addDocuments([
+      { pageContent: "hello", metadata: { a: createdAt + 1 } },
+      { pageContent: "car", metadata: { a: createdAt } },
+      { pageContent: "adjective", metadata: { a: createdAt } },
+      { pageContent: "hi", metadata: { a: createdAt } },
+    ]);
+
+    // Sleeping for a second to make sure that all the indexing operations are finished.
+    await sleep(1000);
+
+    const results1 = await store.similaritySearchWithScore("hello!", 1);
+    expect(results1).toHaveLength(1);
+
+    expect([results1[0][0]]).toEqual([
+      new Document({ metadata: { a: createdAt + 1 }, pageContent: "hello" }),
+    ]);
+
+    const results2 = await store.similaritySearchWithScore("testing!", 6);
+
+    expect(results2).toHaveLength(4);
+
+    await store.delete({ ids: ids.slice(2) });
+
+    const results3 = await store.similaritySearchWithScore("testing again!", 6);
+
+    expect(results3).toHaveLength(2);
+  });
+
+  test("UpstashVectorStore.fromText", async () => {
+    const vectorStore = await UpstashVectorStore.fromTexts(
+      ["hello there!", "what are you building?", "vectors are great!"],
+      [
+        { id: 1, name: "text1" },
+        { id: 2, name: "text2" },
+        { id: 3, name: "text3" },
+      ],
+      embeddings,
+      { index }
+    );
+
+    // Sleeping for a second to make sure that all the indexing operations are finished.
+    await sleep(1000);
+
+    const results1 = await vectorStore.similaritySearch("vectors are great", 1);
+
+    expect(results1).toEqual([
+      new Document({
+        pageContent: "vectors are great!",
+        metadata: { id: 3, name: "text3" },
+      }),
+    ]);
+  });
+});