Update ingest for Azure (#2843)

* Add initial package.json for chatbot-server

* feat(ui): initialize chatbot UI with React and Vite (#2841)

* feat(ui): initialize chatbot UI with React and Vite

- Added package.json for UI package with scripts and dependencies.
- Created App.css for styling the main application layout.
- Implemented App.tsx as the main component integrating the MongoDB Chatbot UI.
- Added a React SVG asset for branding.
- Created index.css for global styles and theming.
- Set up main.tsx as the entry point for the React application.
- Added vite-env.d.ts for Vite type definitions.
- Configured TypeScript with tsconfig.json and tsconfig.node.json for the UI package.
- Created vite.config.ts for Vite configuration with React plugin.

* fix: update license to Apache-2.0 and refactor import statements for consistency

* feat(ui): initialize chatbot UI with React and Vite

- Added package.json for UI package with scripts and dependencies.
- Created App.css for styling the main application layout.
- Implemented App.tsx as the main component integrating the MongoDB Chatbot UI.
- Added a React SVG asset for branding.
- Created index.css for global styles and theming.
- Set up main.tsx as the entry point for the React application.
- Added vite-env.d.ts for Vite type definitions.
- Configured TypeScript with tsconfig.json and tsconfig.node.json for the UI package.
- Created vite.config.ts for Vite configuration with React plugin.

* fix: update license to Apache-2.0 and refactor import statements for consistency

* feat(ingest): add initial configuration and data sources for chatbot ingestion

* update configs

* feat(ingest): implement data sources for LeafyGreen UI and MongoDB Chatbot Framework

* chore: update pnpm workspace configuration to include 'apps/*' directory

* feat(ingest): update ingest configuration to use Azure OpenAI and add loadEnv utility

* feat(ingest): enhance environment variable loading and add MongoDB chatbot framework data source

* feat(ingest): add data sources for LeafyGreen UI and MongoDB Chatbot Framework, and refactor Azure OpenAI embedder

* feat(ingest): remove MongoDB connection URI from example environment file

* feat: remove ts-node from devDependencies and update package versions in pnpm-lock.yaml

* feat(ingest): add MongoDB Design website data source and integrate into ingest configuration

* Update pnpm-lock.yaml

* Delete package-lock.json

* reset ui

* Update .env.example

* rm wip web loader

---------

Co-authored-by: Terrence Keane <terrence.keane@mongodb.com>

Author: TheSonOfThomp

apps/chatbot-server/.env.example

@@ -1,9 +1,20 @@
 # MongoDB config
-MONGODB_CONNECTION_URI="<MongoDB connection URI with Atlas Vector Search configured on 'embedded_content'>"
+MONGODB_USER=<YOUR_MONGODB_USER>
+MONGODB_PASSWORD=<YOUR_MONGODB_PASSWORD>
+MONGODB_PROJECT_URL=<YOUR_PROJECT_URL>
+MONGODB_APP_NAME=LeafyGreenAI
+
 VECTOR_SEARCH_INDEX_NAME="vector_index" # or whatever your index name is
 MONGODB_DATABASE_NAME="mongodb-chatbot-framework-chatbot" # or whatever your database name is. must contain vector search index.
 
 # OpenAI config
 OPENAI_API_KEY=<OpenAI API key>
 OPENAI_EMBEDDING_MODEL="text-embedding-ada-002" # or other model
 OPENAI_CHAT_COMPLETION_MODEL="gpt-3.5-turbo" # or other model
+
+# Azure OpenAI config
+AZURE_API_KEY1=<YOUR_AZURE_API_KEY1>
+AZURE_API_KEY2=<YOUR_AZURE_API_KEY2>
+AZURE_OPENAI_ENDPOINT=https://<your-instance>.openai.azure.com/
+AZURE_OPENAI_EMBEDDING_MODEL=text-embedding-ada-002
+AZURE_OPENAI_CHAT_COMPLETION_MODEL=gpt-3.5-turbo

apps/chatbot-server/packages/ingest/src/ingest.config.ts renamed to apps/chatbot-server/ingest/ingest.config.ts

@@ -1,36 +1,32 @@
 import {
   makeMongoDbEmbeddedContentStore,
   makeMongoDbPageStore,
-  makeOpenAiEmbedder,
 } from 'mongodb-rag-core';
-import { OpenAI } from 'mongodb-rag-core/openai';
+import { AzureOpenAI } from 'mongodb-rag-core/openai';
 import { Config, makeIngestMetaStore } from 'mongodb-rag-ingest';
-import path from 'path';
 
-import { leafygreenGithubSourceConstructor } from './LGGithubDataSource';
-import { loadEnvVars } from './loadEnvVars';
-import { mongoDbChatbotFrameworkDocsDataSourceConstructor } from './mongodbChatbotFrameworkDataSource';
+import { leafygreenGithubSourceConstructor } from './sources/github-leafygreen-ui';
+import { mongoDbChatbotFrameworkDocsDataSourceConstructor } from './sources/github-mdb-chatbot-framework';
+import { createAzureEmbedderConstructor } from './utils/createAzureEmbedderConstructor';
+import { loadEnvVars } from './utils/loadEnv';
 
 // Load project environment variables
-const dotenvPath = path.join(__dirname, '..', '..', '..', '.env'); // .env at project root
 const {
   MONGODB_CONNECTION_URI,
   MONGODB_DATABASE_NAME,
-  OPENAI_API_KEY,
   OPENAI_EMBEDDING_MODEL,
-} = loadEnvVars(dotenvPath);
+} = loadEnvVars();
 
 export default {
-  embedder: async () => {
-    return makeOpenAiEmbedder({
-      openAiClient: new OpenAI({ apiKey: OPENAI_API_KEY }),
-      deployment: OPENAI_EMBEDDING_MODEL,
-      backoffOptions: {
-        numOfAttempts: 25,
-        startingDelay: 1000,
-      },
-    });
-  },
+  embedder: createAzureEmbedderConstructor({
+    azureClient: new AzureOpenAI({
+      endpoint: process.env.AZURE_OPENAI_ENDPOINT,
+      apiKey: process.env.AZURE_API_KEY1,
+      apiVersion: '2024-04-01-preview',
+      deployment: process.env.AZURE_OPENAI_DEPLOYMENT,
+    }),
+    model: OPENAI_EMBEDDING_MODEL,
+  }),
   embeddedContentStore: () =>
     makeMongoDbEmbeddedContentStore({
       connectionUri: MONGODB_CONNECTION_URI,
@@ -52,11 +48,9 @@ export default {
     }),
   // Add data sources here
   dataSources: async () => {
-    const mongodbChatbotFrameworkSource =
-      await mongoDbChatbotFrameworkDocsDataSourceConstructor();
-
-    const leafyGreenGithubSource = await leafygreenGithubSourceConstructor();
-
-    return [mongodbChatbotFrameworkSource, leafyGreenGithubSource];
+    return Promise.all([
+      mongoDbChatbotFrameworkDocsDataSourceConstructor(),
+      leafygreenGithubSourceConstructor(),
+    ]);
   },
 } satisfies Config;

apps/chatbot-server/packages/ingest/src/LGGithubDataSource.ts renamed to apps/chatbot-server/ingest/sources/github-leafygreen-ui.ts

@@ -4,6 +4,25 @@ import {
   makeGitDataSource,
 } from 'mongodb-rag-core/dataSources';
 
+export const leafygreenGithubSourceConstructor = async () => {
+  return await makeGitDataSource({
+    name: 'leafygreen-ui',
+    repoUri: 'https://github.com/mongodb/leafygreen-ui.git',
+    repoOptions: {
+      '--depth': 1,
+      '--branch': 'main',
+    },
+    metadata: {
+      productName: 'LeafyGreen UI',
+      version: '1.0.0',
+      tags: ['leafygreen', 'docs'],
+    },
+    filter: (path: string) => path.endsWith('.md') || path.includes('types'),
+    handlePage: async (path, content) =>
+      await handleHtmlDocument(path, content, htmlParserOptions),
+  });
+};
+
 const removeElements = (domDoc: Document) => [
   ...Array.from(domDoc.querySelectorAll('head')),
   ...Array.from(domDoc.querySelectorAll('script')),
@@ -38,22 +57,3 @@ const htmlParserOptions: Omit<HandleHtmlPageFuncOptions, 'sourceName'> = {
   extractTitle,
   extractMetadata,
 };
-
-export const leafygreenGithubSourceConstructor = async () => {
-  return await makeGitDataSource({
-    name: 'leafygreen-ui',
-    repoUri: 'https://github.com/mongodb/leafygreen-ui.git',
-    repoOptions: {
-      '--depth': 1,
-      '--branch': 'main',
-    },
-    metadata: {
-      productName: 'LeafyGreen UI',
-      version: '1.0.0',
-      tags: ['leafygreen', 'docs'],
-    },
-    filter: (path: string) => path.endsWith('.md') || path.includes('types'),
-    handlePage: async (path, content) =>
-      await handleHtmlDocument(path, content, htmlParserOptions),
-  });
-};

apps/chatbot-server/packages/ingest/src/mongodbChatbotFrameworkDataSource.ts renamed to apps/chatbot-server/ingest/sources/github-mdb-chatbot-framework.ts

@@ -1,3 +1,6 @@
+/**
+  @fileoverview Data source for the MongoDB Chatbot Framework docs.
+ */
 import {
   makeMdOnGithubDataSource,
   MakeMdOnGithubDataSourceParams,

apps/chatbot-server/ingest/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "@lg-tools/build/config/package.tsconfig.json",
+  "compilerOptions": {
+    "module": "NodeNext",
+    "moduleResolution": "nodenext",
+    "target": "ES2017",
+    "rootDir": ".",
+    "outDir": "../dist",
+    "emitDeclarationOnly": false,
+    "composite": false,
+    "declaration": false,
+    "declarationMap": false,
+    "noEmit": false,
+  },
+  "files": [
+    "./ingest.config.ts"
+  ],
+  "include": ["./**/*.ts"],
+}

apps/chatbot-server/ingest/utils/createAzureEmbedderConstructor.ts

@@ -0,0 +1,35 @@
+import { EmbedArgs, Embedder } from 'mongodb-rag-core';
+import { AzureOpenAI } from 'mongodb-rag-core/openai';
+import { Constructor } from 'mongodb-rag-ingest';
+
+/**
+ * Returns a constructor function for an embedder that uses Azure OpenAI
+ * to create embeddings.
+ * @returns
+ */
+export const createAzureEmbedderConstructor =
+  ({
+    azureClient,
+    model,
+  }: {
+    azureClient: AzureOpenAI;
+    model: string;
+  }): Constructor<Embedder> =>
+  () => {
+    const embed: Embedder['embed'] = async (args: EmbedArgs) => {
+      const result = await azureClient.embeddings.create({
+        input: args.text,
+        model,
+      });
+
+      return {
+        // @ts-expect-error
+        embedding: result.data as Array<number>,
+      };
+    };
+
+    return {
+      embed,
+      modelName: model,
+    };
+  };

apps/chatbot-server/ingest/utils/loadEnv.ts

@@ -0,0 +1,49 @@
+import { strict as assert } from 'assert';
+import dotenv from 'dotenv';
+
+/**
+  Load environment variables from a .env file at the given path.
+  Note that if you change the environment variable names,
+  you need to update this function to support those environment variables.
+ */
+export function loadEnvVars() {
+  dotenv.config();
+  const {
+    MONGODB_USER,
+    MONGODB_PASSWORD,
+    MONGODB_PROJECT_URL,
+    MONGODB_APP_NAME,
+    MONGODB_DATABASE_NAME,
+    VECTOR_SEARCH_INDEX_NAME,
+    OPENAI_API_KEY,
+    OPENAI_EMBEDDING_MODEL,
+  } = process.env;
+  const requiredEnvVars = {
+    MONGODB_USER,
+    MONGODB_PASSWORD,
+    MONGODB_PROJECT_URL,
+    MONGODB_APP_NAME,
+    MONGODB_DATABASE_NAME,
+    VECTOR_SEARCH_INDEX_NAME,
+    OPENAI_API_KEY,
+    OPENAI_EMBEDDING_MODEL,
+  } as const;
+
+  for (const [name, value] of Object.entries(requiredEnvVars)) {
+    assert(value, `${name} is required`);
+  }
+
+  const MONGODB_CONNECTION_URI = `mongodb+srv://${MONGODB_USER}:${MONGODB_PASSWORD}@${MONGODB_PROJECT_URL}/?retryWrites=true&w=majority&appName=${MONGODB_APP_NAME}`;
+
+  return {
+    MONGODB_CONNECTION_URI,
+    MONGODB_USER,
+    MONGODB_PASSWORD,
+    MONGODB_PROJECT_URL,
+    MONGODB_APP_NAME,
+    MONGODB_DATABASE_NAME,
+    VECTOR_SEARCH_INDEX_NAME,
+    OPENAI_API_KEY,
+    OPENAI_EMBEDDING_MODEL,
+  } as Record<string, string>;
+}

apps/chatbot-server/package.json

@@ -3,23 +3,25 @@
   "version": "0.0.1",
   "description": "",
   "main": "index.js",
-  "workspaces": [
-    "packages/*"
-  ],
   "publishConfig": {
     "access": "restricted"
   },
   "scripts": {
-    "ingest:all": "npm run ingest:all --workspace=ingest",
+    "build:ingest": "tsc -p ./ingest",
+    "ingest": "pnpm build:ingest && ingest all --config ./dist/ingest.config.js",
     "dev": "npm run dev --workspace=server & npm run dev --workspace=ui"
   },
   "keywords": [],
   "author": "",
   "license": "Apache-2.0",
-  "devDependencies": {
-    "ts-node": "^10.9.2"
-  },
   "dependencies": {
+    "@emotion/css": "^11.13.5",
+    "dotenv": "^16.5.0",
     "jsdom": "^26.1.0"
+  },
+  "devDependencies": {
+    "mongodb-rag-core": "^0.6.3",
+    "mongodb-rag-ingest": "^0.3.1",
+    "tsx": "^4.19.4"
   }
 }