✨ feat: support server db mode with Postgres / Drizzle ORM / tRPC (lobehub#2556)

Author: arvinxx

.github/workflows/test.yml

@@ -1,9 +1,23 @@
 name: Test CI
+
 on: [push, pull_request]
+
 jobs:
   test:
     runs-on: ubuntu-latest
 
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s  --health-retries 5
+
+
+        ports:
+          - 5432:5432
+
     steps:
       - uses: actions/checkout@v4
 
@@ -18,10 +32,27 @@ jobs:
       - name: Lint
         run: bun run lint
 
-      - name: Test and coverage
-        run: bun run test:coverage
+      - name: Test Server Coverage
+        run: bun run test-server:coverage
+        env:
+          DATABASE_TEST_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DATABASE_DRIVER: node
+          NEXT_PUBLIC_SERVICE_MODE: server
+          KEY_VAULTS_SECRET: LA7n9k3JdEcbSgml2sxfw+4TV1AzaaFU5+R176aQz4s=
+
+      - name: Upload Server coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage/server/lcov.info
+          flags: server
+
+      - name: Test App Coverage
+        run: bun run test-app:coverage
 
-      - name: Upload coverage to Codecov
+      - name: Upload App Coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
-          token: ${{ secrets.CODECOV_TOKEN }} # required
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage/app/lcov.info
+          flags: app

codecov.yml

@@ -0,0 +1,11 @@
+coverage:
+  status:
+    project:
+      default: off
+      server:
+        flags:
+          - server
+      app:
+        flags:
+          - app
+    patch: off

drizzle.config.ts

@@ -0,0 +1,29 @@
+import * as dotenv from 'dotenv';
+import type { Config } from 'drizzle-kit';
+
+// Read the .env file if it exists, or a file specified by the
+
+// dotenv_config_path parameter that's passed to Node.js
+
+dotenv.config();
+
+let connectionString = process.env.DATABASE_URL;
+
+if (process.env.NODE_ENV === 'test') {
+  console.log('current ENV:', process.env.NODE_ENV);
+  connectionString = process.env.DATABASE_TEST_URL;
+}
+
+if (!connectionString)
+  throw new Error('`DATABASE_URL` or `DATABASE_TEST_URL` not found in environment');
+
+export default {
+  dbCredentials: {
+    url: connectionString,
+  },
+  dialect: 'postgresql',
+  out: './src/database/server/migrations',
+
+  schema: './src/database/server/schemas/lobechat.ts',
+  strict: true,
+} satisfies Config;

next.config.mjs

@@ -60,6 +60,9 @@ const nextConfig = {
       },
     });
 
+    // https://github.com/pinojs/pino/issues/688#issuecomment-637763276
+    config.externals.push('pino-pretty');
+
     return config;
   },
 };

package.json

@@ -27,10 +27,17 @@
   "sideEffects": false,
   "scripts": {
     "build": "next build",
-    "postbuild": "npm run build-sitemap",
+    "postbuild": "npm run build-sitemap && npm run build-migrate-db",
+    "build-migrate-db": "bun run db:migrate",
     "build-sitemap": "next-sitemap --config next-sitemap.config.mjs",
     "build:analyze": "ANALYZE=true next build",
     "build:docker": "DOCKER=true next build && npm run build-sitemap",
+    "db:generate": "drizzle-kit generate -- dotenv_config_path='.env'",
+    "db:migrate": "MIGRATION_DB=1 tsx scripts/migrateServerDB/index.ts",
+    "db:push": "drizzle-kit push -- dotenv_config_path='.env'",
+    "db:push-test": "NODE_ENV=test drizzle-kit push -- dotenv_config_path='.env'",
+    "db:studio": "drizzle-kit studio",
+    "db:z-pull": "drizzle-kit introspect -- dotenv_config_path='.env'",
     "dev": "next dev -p 3010",
     "dev:clerk-proxy": "ngrok http http://localhost:3011",
     "docs:i18n": "lobe-i18n md && npm run workflow:docs && npm run lint:mdx",
@@ -48,8 +55,11 @@
     "release": "semantic-release",
     "start": "next start",
     "stylelint": "stylelint \"src/**/*.{js,jsx,ts,tsx}\" --fix",
-    "test": "vitest",
-    "test:coverage": "vitest run --coverage",
+    "test": "npm run test-app && npm run test-server",
+    "test-app": "vitest run --config vitest.config.ts",
+    "test-app:coverage": "vitest run --config vitest.config.ts  --coverage",
+    "test-server": "vitest run --config vitest.server.config.ts",
+    "test-server:coverage": "vitest run --config vitest.server.config.ts --coverage",
     "test:update": "vitest -u",
     "type-check": "tsc --noEmit",
     "workflow:docs": "tsx scripts/docsWorkflow/index.ts",
@@ -101,6 +111,7 @@
     "@lobehub/tts": "^1.24.1",
     "@lobehub/ui": "^1.141.2",
     "@microsoft/fetch-event-source": "^2.0.1",
+    "@neondatabase/serverless": "^0.9.3",
     "@next/third-parties": "^14.2.3",
     "@sentry/nextjs": "^7.116.0",
     "@t3-oss/env-nextjs": "^0.10.1",
@@ -119,6 +130,8 @@
     "debug": "^4.3.4",
     "dexie": "^3.2.7",
     "diff": "^5.2.0",
+    "drizzle-orm": "^0.30.10",
+    "drizzle-zod": "^0.5.1",
     "fast-deep-equal": "^3.1.3",
     "gpt-tokenizer": "^2.1.2",
     "i18next": "^23.11.5",
@@ -141,6 +154,7 @@
     "nuqs": "^1.17.4",
     "ollama": "^0.5.1",
     "openai": "^4.47.1",
+    "pg": "^8.11.5",
     "pino": "^9.1.0",
     "polished": "^4.3.1",
     "posthog-js": "^1.135.2",
@@ -163,6 +177,7 @@
     "semver": "^7.6.2",
     "sharp": "^0.33.4",
     "superjson": "^2.2.1",
+    "svix": "^1.24.0",
     "swr": "^2.2.5",
     "systemjs": "^6.15.1",
     "ts-md5": "^1.3.1",
@@ -171,6 +186,7 @@
     "use-merge-value": "^1.2.0",
     "utility-types": "^3.11.0",
     "uuid": "^10.0.0",
+    "ws": "^8.17.0",
     "y-protocols": "^1.0.6",
     "y-webrtc": "^10.3.0",
     "yaml": "^2.4.2",
@@ -200,19 +216,23 @@
     "@types/lodash-es": "^4.17.12",
     "@types/node": "^20.12.12",
     "@types/numeral": "^2.0.5",
+    "@types/pg": "^8.11.6",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "@types/rtl-detect": "^1.0.3",
     "@types/semver": "^7.5.8",
     "@types/systemjs": "^6.13.5",
     "@types/ua-parser-js": "^0.7.39",
     "@types/uuid": "^9.0.8",
+    "@types/ws": "^8.5.10",
     "@umijs/lint": "^4.2.5",
     "@vitest/coverage-v8": "~1.2.2",
     "ajv-keywords": "^5.1.0",
     "commitlint": "^19.3.0",
     "consola": "^3.2.3",
+    "dotenv": "^16.4.5",
     "dpdm": "^3.14.0",
+    "drizzle-kit": "^0.21.1",
     "eslint": "^8.57.0",
     "eslint-plugin-mdx": "^2.3.4",
     "fake-indexeddb": "^6.0.0",

scripts/migrateServerDB/index.ts

@@ -0,0 +1,30 @@
+import * as dotenv from 'dotenv';
+import * as migrator from 'drizzle-orm/neon-serverless/migrator';
+import { join } from 'node:path';
+
+import { serverDB } from '../../src/database/server/core/db';
+
+// Read the `.env` file if it exists, or a file specified by the
+// dotenv_config_path parameter that's passed to Node.js
+dotenv.config();
+
+const runMigrations = async () => {
+  await migrator.migrate(serverDB, {
+    migrationsFolder: join(__dirname, '../../src/database/server/migrations'),
+  });
+  console.log('✅ database migration pass.');
+  // eslint-disable-next-line unicorn/no-process-exit
+  process.exit(0);
+};
+
+let connectionString = process.env.DATABASE_URL;
+
+// only migrate database if the connection string is available
+if (connectionString) {
+  // eslint-disable-next-line unicorn/prefer-top-level-await
+  runMigrations().catch((err) => {
+    console.error('❌ Database migrate failed:', err);
+    // eslint-disable-next-line unicorn/no-process-exit
+    process.exit(1);
+  });
+}

src/app/(main)/(mobile)/me/(home)/features/useCategory.tsx

@@ -6,6 +6,7 @@ import { useTranslation } from 'react-i18next';
 import { CellProps } from '@/components/Cell';
 import { enableAuth } from '@/const/auth';
 import { DISCORD, DOCUMENTS, FEEDBACK } from '@/const/url';
+import { isServerMode } from '@/const/version';
 import { usePWAInstall } from '@/hooks/usePWAInstall';
 import { useUserStore } from '@/store/user';
 import { authSelectors } from '@/store/user/slices/auth/selectors';
@@ -109,7 +110,7 @@ export const useCategory = () => {
 
     /* ↑ cloud slot ↑ */
     ...(canInstall ? pwa : []),
-    ...(isLogin ? data : []),
+    ...(isLogin && !isServerMode ? data : []),
     ...helps,
   ].filter(Boolean) as CellProps[];