🐛 fix: remove orphan chunks if there is no related file (#3578)

* ✅ test: add more tests * 🐛 fix: add `DISABLE_REMOVE_GLOBAL_FILE` in dbENV * 🐛 fix: remove orphan chunks if there is related file * ✅ test: fix test * 👷 fix: throw error when not set `APP_URL` at server side * ✅ test: fix test
lobehub · Aug 23, 2024 · 36bcaf3 · 36bcaf3
1 parent 6d96a41
commit 36bcaf3
Show file tree

Hide file tree

Showing 11 changed files with 479 additions and 81 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -18,7 +18,7 @@ jobs:
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s  --health-retries 5
         ports:
           - 5432:5432
-          
+
     steps:
       - uses: actions/checkout@v4
 
@@ -40,7 +40,8 @@ jobs:
           DATABASE_DRIVER: node
           NEXT_PUBLIC_SERVICE_MODE: server
           KEY_VAULTS_SECRET: LA7n9k3JdEcbSgml2sxfw+4TV1AzaaFU5+R176aQz4s=
-          NEXT_PUBLIC_S3_DOMAIN: https://example.com
+          S3_PUBLIC_DOMAIN: https://example.com
+          APP_URL: https://home.com
 
       - name: Test App Coverage
         run: bun run test-app:coverage

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -39,7 +39,8 @@ jobs:
           DATABASE_DRIVER: node
           NEXT_PUBLIC_SERVICE_MODE: server
           KEY_VAULTS_SECRET: LA7n9k3JdEcbSgml2sxfw+4TV1AzaaFU5+R176aQz4s=
-          NEXT_PUBLIC_S3_DOMAIN: https://example.com
+          S3_PUBLIC_DOMAIN: https://example.com
+          APP_URL: https://home.com
 
       - name: Upload Server coverage to Codecov
         uses: codecov/codecov-action@v4

diff --git a/src/config/app.ts b/src/config/app.ts
@@ -2,6 +2,8 @@
 import { createEnv } from '@t3-oss/env-nextjs';
 import { z } from 'zod';
 
+import { isServerMode } from '@/const/version';
+
 declare global {
   // eslint-disable-next-line @typescript-eslint/no-namespace
   namespace NodeJS {
@@ -10,14 +12,19 @@ declare global {
     }
   }
 }
+const isInVercel = process.env.VERCEL === '1';
 
-export const getAppConfig = () => {
-  const ACCESS_CODES = process.env.ACCESS_CODE?.split(',').filter(Boolean) || [];
-  const isInVercel = process.env.VERCEL === '1';
+const vercelUrl = `https://${process.env.VERCEL_URL}`;
 
-  const vercelUrl = `https://${process.env.VERCEL_URL}`;
+const APP_URL = process.env.APP_URL ? process.env.APP_URL : isInVercel ? vercelUrl : undefined;
 
-  const APP_URL = process.env.APP_URL ? process.env.APP_URL : isInVercel ? vercelUrl : undefined;
+// only throw error in server mode and server side
+if (typeof window === 'undefined' && isServerMode && !APP_URL) {
+  throw new Error('`APP_URL` is required in server mode');
+}
+
+export const getAppConfig = () => {
+  const ACCESS_CODES = process.env.ACCESS_CODE?.split(',').filter(Boolean) || [];
 
   return createEnv({
     client: {

diff --git a/src/config/db.ts b/src/config/db.ts
@@ -11,6 +11,8 @@ export const getServerDBConfig = () => {
       DATABASE_TEST_URL: process.env.DATABASE_TEST_URL,
       DATABASE_URL: process.env.DATABASE_URL,
 
+      DISABLE_REMOVE_GLOBAL_FILE: process.env.DISABLE_REMOVE_GLOBAL_FILE === '1',
+
       KEY_VAULTS_SECRET: process.env.KEY_VAULTS_SECRET,
 
       NEXT_PUBLIC_ENABLED_SERVER_SERVICE: process.env.NEXT_PUBLIC_SERVICE_MODE === 'server',
@@ -20,6 +22,8 @@ export const getServerDBConfig = () => {
       DATABASE_TEST_URL: z.string().optional(),
       DATABASE_URL: z.string().optional(),
 
+      DISABLE_REMOVE_GLOBAL_FILE: z.boolean().optional(),
+
       KEY_VAULTS_SECRET: z.string().optional(),
     },
   });

diff --git a/src/database/server/models/__tests__/chunk.test.ts b/src/database/server/models/__tests__/chunk.test.ts
@@ -98,7 +98,66 @@ describe('ChunkModel', () => {
     });
   });
 
-  // Add more test cases for other methods...
+  describe('deleteOrphanChunks', () => {
+    it('should delete orphaned chunks', async () => {
+      // Create orphaned chunks
+      await serverDB
+        .insert(chunks)
+        .values([
+          { text: 'Orphan Chunk 1', userId },
+          { text: 'Orphan Chunk 2', userId },
+        ])
+        .returning();
+
+      // Create a non-orphaned chunk
+      const [nonOrphanChunk] = await serverDB
+        .insert(chunks)
+        .values([{ text: 'Non-Orphan Chunk', userId }])
+        .returning();
+
+      await serverDB.insert(fileChunks).values([{ fileId: '1', chunkId: nonOrphanChunk.id }]);
+
+      // Execute the method
+      await chunkModel.deleteOrphanChunks();
+
+      // Check if orphaned chunks are deleted
+      const remainingChunks = await serverDB.query.chunks.findMany();
+      expect(remainingChunks).toHaveLength(1);
+      expect(remainingChunks[0].id).toBe(nonOrphanChunk.id);
+    });
+
+    it('should not delete any chunks when there are no orphans', async () => {
+      // Create non-orphaned chunks
+      const [chunk1, chunk2] = await serverDB
+        .insert(chunks)
+        .values([
+          { text: 'Chunk 1', userId },
+          { text: 'Chunk 2', userId },
+        ])
+        .returning();
+
+      await serverDB.insert(fileChunks).values([
+        { fileId: '1', chunkId: chunk1.id },
+        { fileId: '2', chunkId: chunk2.id },
+      ]);
+
+      // Execute the method
+      await chunkModel.deleteOrphanChunks();
+
+      // Check if all chunks are still present
+      const remainingChunks = await serverDB.query.chunks.findMany();
+      expect(remainingChunks).toHaveLength(2);
+    });
+
+    it('should not throw an error when the database is empty', async () => {
+      // Ensure the database is empty
+      await serverDB.delete(chunks);
+      await serverDB.delete(fileChunks);
+
+      // Execute the method and expect it not to throw
+      await expect(chunkModel.deleteOrphanChunks()).resolves.not.toThrow();
+    });
+  });
 
   describe('semanticSearch', () => {
     it('should perform semantic search and return results', async () => {