Merge branch 'danny-avila:main' into Refresh-Token-Intercept

api/lib/db/indexSync.js

@@ -1,6 +1,5 @@
-const mongoose = require('mongoose');
-const Conversation = mongoose.models.Conversation;
-const Message = mongoose.models.Message;
+const Conversation = require('../../models/schema/convoSchema');
+const Message = require('../../models/schema/messageSchema');
 const { MeiliSearch } = require('meilisearch');
 let currentTimeout = null;
 
@@ -37,12 +36,12 @@ async function indexSync(req, res, next) {
 
     if (messageCount !== messagesIndexed) {
       console.log('Messages out of sync, indexing');
-      await Message.syncWithMeili();
+      Message.syncWithMeili();
     }
 
     if (convoCount !== convosIndexed) {
       console.log('Convos out of sync, indexing');
-      await Conversation.syncWithMeili();
+      Conversation.syncWithMeili();
     }
   } catch (err) {
     // console.log('in index sync');

api/models/Conversation.js

@@ -54,8 +54,6 @@ module.exports = {
       const cache = {};
       const convoMap = {};
       const promises = [];
-      // will handle a syncing solution soon
-      const deletedConvoIds = [];
 
       convoIds.forEach((convo) =>
         promises.push(
@@ -66,23 +64,17 @@ module.exports = {
         ),
       );
 
-      const results = (await Promise.all(promises)).filter((convo, i) => {
-        if (!convo) {
-          deletedConvoIds.push(convoIds[i].conversationId);
-          return false;
-        } else {
-          const page = Math.floor(i / pageSize) + 1;
-          if (!cache[page]) {
-            cache[page] = [];
-          }
-          cache[page].push(convo);
-          convoMap[convo.conversationId] = convo;
-          return true;
+      const results = (await Promise.all(promises)).filter(Boolean);
+
+      results.forEach((convo, i) => {
+        const page = Math.floor(i / pageSize) + 1;
+        if (!cache[page]) {
+          cache[page] = [];
         }
+        cache[page].push(convo);
+        convoMap[convo.conversationId] = convo;
       });
 
-      // const startIndex = (pageNumber - 1) * pageSize;
-      // const convos = results.slice(startIndex, startIndex + pageSize);
       const totalPages = Math.ceil(results.length / pageSize);
       cache.pages = totalPages;
       cache.pageSize = pageSize;
@@ -92,8 +84,6 @@ module.exports = {
         pages: totalPages || 1,
         pageNumber,
         pageSize,
-        // will handle a syncing solution soon
-        filter: new Set(deletedConvoIds),
         convoMap,
       };
     } catch (error) {

api/models/plugins/mongoMeili.js

@@ -14,35 +14,116 @@ const validateOptions = function (options) {
   });
 };
 
-const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) {
-  // console.log('attributesToIndex', attributesToIndex);
+// const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) {
+const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
   const primaryKey = attributesToIndex[0];
   // MeiliMongooseModel is of type Mongoose.Model
   class MeiliMongooseModel {
-    // Clear Meili index
-    static async clearMeiliIndex() {
-      await index.delete();
-      // await index.deleteAllDocuments();
-      await this.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } });
-    }
-
-    static async resetIndex() {
-      await this.clearMeiliIndex();
-      await client.createIndex(indexName, { primaryKey });
-    }
-    // Clear Meili index
-    // Push a mongoDB collection to Meili index
+    /**
+     * `syncWithMeili`: synchronizes the data between a MongoDB collection and a MeiliSearch index,
+     * only triggered if there's ever a discrepancy determined by `api\lib\db\indexSync.js`.
+     *
+     * 1. Fetches all documents from the MongoDB collection and the MeiliSearch index.
+     * 2. Compares the documents from both sources.
+     * 3. If a document exists in MeiliSearch but not in MongoDB, it's deleted from MeiliSearch.
+     * 4. If a document exists in MongoDB but not in MeiliSearch, it's added to MeiliSearch.
+     * 5. If a document exists in both but has different `text` or `title` fields (depending on the `primaryKey`), it's updated in MeiliSearch.
+     * 6. After all operations, it updates the `_meiliIndex` field in MongoDB to indicate whether the document is indexed in MeiliSearch.
+     *
+     * Note: This strategy does not use batch operations for Meilisearch as the `index.addDocuments` will discard
+     * the entire batch if there's an error with one document, and will not throw an error if there's an issue.
+     * Also, `index.getDocuments` needs an exact limit on the amount of documents to return, so we build the map in batches.
+     *
+     * @returns {Promise} A promise that resolves when the synchronization is complete.
+     *
+     * @throws {Error} Throws an error if there's an issue with adding a document to MeiliSearch.
+     */
     static async syncWithMeili() {
-      await this.resetIndex();
-      const docs = await this.find({ _meiliIndex: { $in: [null, false] } });
-      console.log('docs', docs.length);
-      const objs = docs.map((doc) => doc.preprocessObjectForIndex());
       try {
-        await index.addDocuments(objs);
-        const ids = docs.map((doc) => doc._id);
-        await this.collection.updateMany({ _id: { $in: ids } }, { $set: { _meiliIndex: true } });
+        let moreDocuments = true;
+        const mongoDocuments = await this.find().lean();
+        const format = (doc) => _.pick(doc, attributesToIndex);
+
+        // Prepare for comparison
+        const mongoMap = new Map(mongoDocuments.map((doc) => [doc[primaryKey], format(doc)]));
+        const indexMap = new Map();
+        let offset = 0;
+        const batchSize = 1000;
+
+        while (moreDocuments) {
+          const batch = await index.getDocuments({ limit: batchSize, offset });
+
+          if (batch.results.length === 0) {
+            moreDocuments = false;
+          }
+
+          for (const doc of batch.results) {
+            indexMap.set(doc[primaryKey], format(doc));
+          }
+
+          offset += batchSize;
+        }
+
+        console.log('indexMap', indexMap.size);
+        console.log('mongoMap', mongoMap.size);
+
+        const updateOps = [];
+
+        // Iterate over Meili index documents
+        for (const [id, doc] of indexMap) {
+          const update = {};
+          update[primaryKey] = id;
+          if (mongoMap.has(id)) {
+            // Case: Update
+            // If document also exists in MongoDB, would be update case
+            if (
+              (doc.text && doc.text !== mongoMap.get(id).text) ||
+              (doc.title && doc.title !== mongoMap.get(id).title)
+            ) {
+              console.log(`${id} had document discrepancy in ${doc.text ? 'text' : 'title'} field`);
+              updateOps.push({
+                updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
+              });
+              await index.addDocuments([doc]);
+            }
+          } else {
+            // Case: Delete
+            // If document does not exist in MongoDB, its a delete case from meili index
+            await index.deleteDocument(id);
+            updateOps.push({
+              updateOne: { filter: update, update: { $set: { _meiliIndex: false } } },
+            });
+          }
+        }
+
+        // Iterate over MongoDB documents
+        for (const [id, doc] of mongoMap) {
+          const update = {};
+          update[primaryKey] = id;
+          // Case: Insert
+          // If document does not exist in Meili Index, Its an insert case
+          if (!indexMap.has(id)) {
+            await index.addDocuments([doc]);
+            updateOps.push({
+              updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
+            });
+          } else if (doc._meiliIndex === false) {
+            updateOps.push({
+              updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
+            });
+          }
+        }
+
+        if (updateOps.length > 0) {
+          await this.collection.bulkWrite(updateOps);
+          console.log(
+            `[Meilisearch] Finished indexing ${
+              primaryKey === 'messageId' ? 'messages' : 'conversations'
+            }`,
+          );
+        }
       } catch (error) {
-        console.log('Error adding document to Meili');
+        console.log('[Meilisearch] Error adding document to Meili');
         console.error(error);
       }
     }
@@ -72,7 +153,7 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
             },
             { _id: 1 },
           ),
-        );
+        ).lean();
 
         // Add additional data from mongodb into Meili search hits
         const populatedHits = data.hits.map(function (hit) {
@@ -81,7 +162,7 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
           const originalHit = _.find(hitsFromMongoose, query);
 
           return {
-            ...(originalHit ? originalHit.toJSON() : {}),
+            ...(originalHit ?? {}),
             ...hit,
           };
         });

api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/backend",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "description": "",
   "scripts": {
     "start": "echo 'please run this from the root directory'",

api/server/routes/ask/anthropic.js

@@ -10,7 +10,11 @@ const { handleError, sendMessage, createOnProgress } = require('./handlers');
 const abortControllers = new Map();
 
 router.post('/abort', requireJwtAuth, async (req, res) => {
-  return await abortMessage(req, res, abortControllers);
+  try {
+    return await abortMessage(req, res, abortControllers);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
 router.post('/', requireJwtAuth, async (req, res) => {

api/server/routes/ask/gptPlugins.js

@@ -15,7 +15,11 @@ const requireJwtAuth = require('../../../middleware/requireJwtAuth');
 const abortControllers = new Map();
 
 router.post('/abort', requireJwtAuth, async (req, res) => {
-  return await abortMessage(req, res, abortControllers);
+  try {
+    return await abortMessage(req, res, abortControllers);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
 router.post('/', requireJwtAuth, async (req, res) => {

api/server/routes/ask/openAI.js

@@ -9,7 +9,11 @@ const requireJwtAuth = require('../../../middleware/requireJwtAuth');
 const abortControllers = new Map();
 
 router.post('/abort', requireJwtAuth, async (req, res) => {
-  return await abortMessage(req, res, abortControllers);
+  try {
+    return await abortMessage(req, res, abortControllers);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
 router.post('/', requireJwtAuth, async (req, res) => {

api/server/routes/convos.js

@@ -27,7 +27,8 @@ router.post('/clear', requireJwtAuth, async (req, res) => {
     filter = { conversationId };
   }
 
-  console.log('source:', source);
+  // for debugging deletion source
+  // console.log('source:', source);
 
   if (source === 'button' && !conversationId) {
     return res.status(200).send('No conversationId provided');

api/server/routes/search.js

@@ -90,11 +90,6 @@ router.get('/', requireJwtAuth, async function (req, res) {
   }
 });
 
-router.get('/clear', async function (req, res) {
-  await Message.resetIndex();
-  res.send('cleared');
-});
-
 router.get('/test', async function (req, res) {
   const { q } = req.query;
   const messages = (

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/frontend",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "description": "",
   "scripts": {
     "data-provider": "cd .. && npm run build:data-provider",

client/src/components/Input/Footer.tsx

@@ -3,6 +3,7 @@ import { useGetStartupConfig } from 'librechat-data-provider';
 
 export default function Footer() {
   const { data: config } = useGetStartupConfig();
+
   return (
     <div className="hidden px-3 pb-1 pt-2 text-center text-xs text-black/50 dark:text-white/50 md:block md:px-4 md:pb-4 md:pt-3">
       <a
@@ -11,10 +12,9 @@ export default function Footer() {
         rel="noreferrer"
         className="underline"
       >
-        {config?.appTitle || 'LibreChat'}
+        {config?.appTitle || 'LibreChat'} v0.5.6
       </a>
-      . Serves and searches all conversations reliably. All AI convos under one house. Pay per call
-      and not per month (cents compared to dollars).
+      {' - '}. All AI conversations in one place. Pay per call and not per month.
     </div>
   );
 }

client/src/localization/languages/Eng.tsx

@@ -2,7 +2,7 @@
 
 export default {
   com_ui_examples: 'Examples',
-  com_ui_new_chat: 'New Chat',
+  com_ui_new_chat: 'New chat',
   com_ui_example_quantum_computing: 'Explain quantum computing in simple terms',
   com_ui_example_10_year_old_b_day: 'Got any creative ideas for a 10 year old\'s birthday?',
   com_ui_example_http_in_js: 'How do I make an HTTP request in Javascript?',

docs/deployment/ngrok.md

@@ -6,24 +6,46 @@ To use Ngrok for tunneling your local server to the internet, follow these steps
 
 1. Go to https://ngrok.com/ and sign up for an account.
 
-## Docker Installation
+## Docker Installation 🐳
 
 1. Copy your auth token from https://dashboard.ngrok.com/get-started/your-authtoken.
 2. Open a terminal and run the following command: `docker run -d -it -e NGROK_AUTHTOKEN=<your token> ngrok/ngrok http 80`
 
-## Windows Installation
+## Windows Installation 💙
 
 1. Download the ZIP file from https://ngrok.com/download.
 2. Extract the contents of the ZIP file using 7zip or WinRar.
-3. Run `ngrok.exe`.
-4. Copy your auth token from https://dashboard.ngrok.com/get-started/your-authtoken.
-5. In the `ngrok.exe` terminal, run the following command: `ngrok config add-authtoken <your token>`
-6. If you haven't done so already, start LibreChat normally.
-7. In the `ngrok.exe` terminal, run the following command: ngrok http 3080
+3. 
+4. Run `ngrok.exe`.
+5. Copy your auth token from https://dashboard.ngrok.com/get-started/your-authtoken.
+6. In the `ngrok.exe` terminal, run the following command: `ngrok config add-authtoken <your token>`
+7. If you haven't done so already, start LibreChat normally.
+8. In the `ngrok.exe` terminal, run the following command: `ngrok http 3080`
 
 You will see a link that can be used to access LibreChat.
 ![ngrok-1](https://github.com/danny-avila/LibreChat/assets/32828263/3cb4b063-541f-4f0a-bea8-a04dd36e6bf4)
 
+## Linux Installation 🐧
+
+1. Copy the command from https://ngrok.com/download choosing the **correct** architecture.
+2. Run the command in the terminal
+3. Copy your auth token from https://dashboard.ngrok.com/get-started/your-authtoken.
+4. run the following command: `ngrok config add-authtoken <your token>`
+5. If you haven't done so already, start LibreChat normally.
+6. run the following command: `ngrok http 3080`
+
+## Mac Installation 🍎
+
+1. Download the ZIP file from https://ngrok.com/download.
+2. Extract the contents of the ZIP file using a suitable Mac application like Unarchiver.
+3. Open Terminal.
+4. Navigate to the directory where you extracted ngrok using the `cd` command.
+5. Run ngrok by typing `./ngrok`.
+6. Copy your auth token from https://dashboard.ngrok.com/get-started/your-authtoken.
+7. In the terminal where you ran ngrok, enter the following command: `ngrok authtoken <your token>`
+8. If you haven't done so already, start LibreChat normally.
+9. In the terminal where you ran ngrok, enter the following command: `./ngrok http 3080`
+
 ---
 
 ### Note: