Migrate to postgres

Dockerfile

@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK
 
 RUN yarn workspace @sourcebot/web build
 
-
 # ------ Build Backend ------
 FROM node-alpine AS backend-builder
 WORKDIR /app
@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
 COPY --from=database-builder /app/packages/db ./packages/db
 RUN yarn workspace @sourcebot/backend install --frozen-lockfile
 RUN yarn workspace @sourcebot/backend build
-
+
+
 # ------ Runner ------
 FROM node-alpine AS runner
 WORKDIR /app
@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
 ENV DATA_DIR=/data
 ENV CONFIG_PATH=$DATA_DIR/config.json
 ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
+ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
+ENV DB_NAME=sourcebot
 
 ARG SOURCEBOT_VERSION=unknown
 ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
 # ENV SOURCEBOT_TELEMETRY_DISABLED=1
 
 # Configure dependencies
-RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
+RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib
 
 # Configure zoekt
 COPY vendor/zoekt/install-ctags-alpine.sh .
 RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
 RUN mkdir -p ${DATA_CACHE_DIR}
 COPY --from=zoekt-builder \
-    /cmd/zoekt-git-index \
-    /cmd/zoekt-indexserver \
-    /cmd/zoekt-mirror-github \
-    /cmd/zoekt-mirror-gitiles \
-    /cmd/zoekt-mirror-bitbucket-server \
-    /cmd/zoekt-mirror-gitlab \
-    /cmd/zoekt-mirror-gerrit \
-    /cmd/zoekt-webserver \
-    /cmd/zoekt-index \
-    /usr/local/bin/
+/cmd/zoekt-git-index \
+/cmd/zoekt-indexserver \
+/cmd/zoekt-mirror-github \
+/cmd/zoekt-mirror-gitiles \
+/cmd/zoekt-mirror-bitbucket-server \
+/cmd/zoekt-mirror-gitlab \
+/cmd/zoekt-mirror-gerrit \
+/cmd/zoekt-webserver \
+/cmd/zoekt-index \
+/usr/local/bin/
 
 # Configure the webapp
 COPY --from=web-builder /app/packages/web/public ./packages/web/public
@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
 COPY --from=backend-builder /app/packages/backend ./packages/backend
 
 # Configure the database
+RUN mkdir -p /run/postgresql && \
+    chown -R postgres:postgres /run/postgresql && \
+    chmod 775 /run/postgresql
 COPY --from=database-builder /app/node_modules ./node_modules
 COPY --from=database-builder /app/packages/db ./packages/db
 
@@ -134,6 +139,7 @@ RUN chmod +x ./entrypoint.sh
 COPY default-config.json .
 
 EXPOSE 3000
+EXPOSE 5432
 ENV PORT=3000
 ENV HOSTNAME="0.0.0.0"
 ENTRYPOINT ["/sbin/tini", "--", "./entrypoint.sh"]

entrypoint.sh

@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
     mkdir -p "$DATA_CACHE_DIR"
 fi
 
-# Run a Database migration
-echo -e "\e[34m[Info] Running database migration...\e[0m"
-export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
-yarn workspace @sourcebot/db prisma:migrate:prod
+# Check if DB_DATA_DIR exists, if not initialize it
+if [ ! -d "$DB_DATA_DIR" ]; then
+    echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
+    mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
+    su postgres -c "initdb -D $DB_DATA_DIR"
+fi
 
 # In order to detect if this is the first run, we create a `.installed` file in
 # the cache directory.
@@ -162,5 +164,27 @@ fi
 # }
 
 
+# Start the database and wait for it to be ready before starting any other service
+su postgres -c "postgres -D $DB_DATA_DIR" &
+until pg_isready -h localhost -p 5432 -U postgres; do
+    echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
+    sleep 1
+done
+
+# Check if the database already exists, and create it if it dne
+EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")
+
+if [ "$EXISTING_DB" = "1" ]; then
+  echo "Database '$DB_NAME' already exists; skipping creation."
+else
+  echo "Creating database '$DB_NAME'..."
+  psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
+fi
+
+# Run a Database migration
+export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
+echo -e "\e[34m[Info] Running database migration...\e[0m"
+yarn workspace @sourcebot/db prisma:migrate:prod
+
 # Run supervisord
 exec supervisord -c /etc/supervisor/conf.d/supervisord.conf

package.json

@@ -6,13 +6,19 @@
     "scripts": {
         "build": "yarn workspaces run build",
         "test": "yarn workspaces run test",
-        "dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
-        "dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
+        "dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
+        "dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
+        "dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
         "dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
-        "dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
         "dev:backend": "yarn workspace @sourcebot/backend dev:watch",
         "dev:web": "yarn workspace @sourcebot/web dev",
-        "dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
+        "dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
+        "dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
+        "dev:deps:stop": "docker stop redis postgres",
+        "dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
+        "dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
+        "dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
+        "dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
     },
     "devDependencies": {
         "cross-env": "^7.0.3",

packages/backend/src/main.ts

@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
                     },
                     data: {
                         syncStatus: ConfigSyncStatus.SYNCED,
+                        syncedAt: new Date()
                     }
                 })
             });

packages/db/.env

@@ -1 +1 @@
-DATABASE_URL=file:../../../.sourcebot/db.sqlite
+DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot