Switch to using t3-env for env-var management (#230)

Author: brendan-kellam

.dockerignore

@@ -1,11 +1,13 @@
 Dockerfile
 .dockerignore
-node_modules
 npm-debug.log
 README.md
-.next
-!.next/static
-!.next/standalone
 .git
 .sourcebot
-.env.local
+packages/web/.next
+!packages/web/.next/static
+!packages/web/.next/standalone
+**/node_modules
+**/.env.local
+**/.sentryclirc
+**/.env.sentry-build-plugin

Dockerfile

@@ -1,5 +1,18 @@
+# ------ Global scope variables ------
+# Set of global build arguments.
+# @see: https://docs.docker.com/build/building/variables/#scoping
+
+ARG SOURCEBOT_VERSION
+# PAPIK = Project API Key
+# Note that this key does not need to be kept secret, so it's not
+# necessary to use Docker build secrets here.
+# @see: https://posthog.com/tutorials/api-capture-events#authenticating-with-the-project-api-key
+ARG POSTHOG_PAPIK
+ARG SENTRY_ENVIRONMENT
+
 FROM node:20-alpine3.19 AS node-alpine
 FROM golang:1.23.4-alpine3.19 AS go-alpine
+# ----------------------------------
 
 # ------ Build Zoekt ------
 FROM go-alpine AS zoekt-builder
@@ -9,6 +22,7 @@ COPY vendor/zoekt/go.mod vendor/zoekt/go.sum ./
 RUN go mod download
 COPY vendor/zoekt ./
 RUN CGO_ENABLED=0 GOOS=linux go build -o /cmd/ ./cmd/...
+# -------------------------
 
 # ------ Build shared libraries ------
 FROM node-alpine AS shared-libs-builder
@@ -23,9 +37,24 @@ RUN yarn workspace @sourcebot/db install --frozen-lockfile
 RUN yarn workspace @sourcebot/schemas install --frozen-lockfile
 RUN yarn workspace @sourcebot/crypto install --frozen-lockfile
 RUN yarn workspace @sourcebot/error install --frozen-lockfile
+# ------------------------------------
 
 # ------ Build Web ------
 FROM node-alpine AS web-builder
+ENV DOCKER_BUILD=1
+# -----------
+# Global args
+ARG SOURCEBOT_VERSION
+ENV NEXT_PUBLIC_SOURCEBOT_VERSION=$SOURCEBOT_VERSION
+ARG POSTHOG_PAPIK
+ENV NEXT_PUBLIC_POSTHOG_PAPIK=$POSTHOG_PAPIK
+ARG SENTRY_ENVIRONMENT
+ENV NEXT_PUBLIC_SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
+# Local args
+ARG SENTRY_WEBAPP_DSN
+ENV NEXT_PUBLIC_SENTRY_WEBAPP_DSN=$SENTRY_WEBAPP_DSN
+# -----------
+
 RUN apk add --no-cache libc6-compat
 WORKDIR /app
 
@@ -43,26 +72,13 @@ RUN yarn config set network-timeout 1200000
 RUN yarn workspace @sourcebot/web install --frozen-lockfile
 
 ENV NEXT_TELEMETRY_DISABLED=1
-# @see: https://phase.dev/blog/nextjs-public-runtime-variables/
-ARG NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED=BAKED_NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED
-ARG NEXT_PUBLIC_SOURCEBOT_VERSION=BAKED_NEXT_PUBLIC_SOURCEBOT_VERSION
-ENV NEXT_PUBLIC_PUBLIC_SEARCH_DEMO=BAKED_NEXT_PUBLIC_PUBLIC_SEARCH_DEMO
-ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK
-ENV NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=BAKED_NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY
-ENV NEXT_PUBLIC_SENTRY_ENVIRONMENT=BAKED_NEXT_PUBLIC_SENTRY_ENVIRONMENT
-ENV NEXT_PUBLIC_SENTRY_WEBAPP_DSN=BAKED_NEXT_PUBLIC_SENTRY_WEBAPP_DSN
-
-# @nocheckin: This was interfering with the the `matcher` regex in middleware.ts,
-# causing regular expressions parsing errors when making a request. It's unclear
-# why exactly this was happening, but it's likely due to a bad replacement happening
-# in the `sed` command.
-# @note: leading "/" is required for the basePath property. @see: https://nextjs.org/docs/app/api-reference/next-config-js/basePath
-# ARG NEXT_PUBLIC_DOMAIN_SUB_PATH=/BAKED_NEXT_PUBLIC_DOMAIN_SUB_PATH
-
 RUN yarn workspace @sourcebot/web build
+ENV DOCKER_BUILD=0
+# ------------------------------
 
 # ------ Build Backend ------
 FROM node-alpine AS backend-builder
+ENV DOCKER_BUILD=1
 WORKDIR /app
 
 COPY package.json yarn.lock* ./
@@ -75,10 +91,22 @@ COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 RUN yarn workspace @sourcebot/backend install --frozen-lockfile
 RUN yarn workspace @sourcebot/backend build
-    
+ENV DOCKER_BUILD=0
+# ------------------------------
 
 # ------ Runner ------
 FROM node-alpine AS runner
+# -----------
+# Global args
+ARG SOURCEBOT_VERSION
+ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
+ARG POSTHOG_PAPIK
+ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
+# Local args
+# -----------
+
+RUN echo "Sourcebot Version: $SOURCEBOT_VERSION"
+
 WORKDIR /app
 ENV NODE_ENV=production
 ENV NEXT_TELEMETRY_DISABLED=1
@@ -90,14 +118,6 @@ ENV DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot"
 ENV REDIS_URL="redis://localhost:6379"
 ENV SRC_TENANT_ENFORCEMENT_MODE=strict
 
-ARG SOURCEBOT_VERSION=unknown
-ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
-RUN echo "Sourcebot Version: $SOURCEBOT_VERSION"
-
-ARG PUBLIC_SEARCH_DEMO=false
-ENV PUBLIC_SEARCH_DEMO=$PUBLIC_SEARCH_DEMO
-RUN echo "Public Search Demo: $PUBLIC_SEARCH_DEMO"
-
 # Valid values are: debug, info, warn, error
 ENV SOURCEBOT_LOG_LEVEL=info
 
@@ -106,18 +126,9 @@ ENV SOURCEBOT_LOG_LEVEL=info
 # will serve from http(s)://example.com/sb
 ENV DOMAIN_SUB_PATH=/
 
-# PAPIK = Project API Key
-# Note that this key does not need to be kept secret, so it's not
-# necessary to use Docker build secrets here.
-# @see: https://posthog.com/tutorials/api-capture-events#authenticating-with-the-project-api-key
-ARG POSTHOG_PAPIK=
-ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
-
 # Sourcebot collects anonymous usage data using [PostHog](https://posthog.com/). Uncomment this line to disable.
 # ENV SOURCEBOT_TELEMETRY_DISABLED=1
 
-ENV STRIPE_PUBLISHABLE_KEY=""
-
 # Configure zoekt
 COPY vendor/zoekt/install-ctags-alpine.sh .
 RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
@@ -178,4 +189,5 @@ COPY default-config.json .
 EXPOSE 3000
 ENV PORT=3000
 ENV HOSTNAME="0.0.0.0"
-ENTRYPOINT ["/sbin/tini", "--", "./entrypoint.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "./entrypoint.sh"]
+# ------------------------------

entrypoint.sh

@@ -108,102 +108,6 @@ fi
 echo "{\"version\": \"$SOURCEBOT_VERSION\", \"install_id\": \"$SOURCEBOT_INSTALL_ID\"}" > "$FIRST_RUN_FILE"
 
 
-# Update NextJs public env variables w/o requiring a rebuild.
-# @see: https://phase.dev/blog/nextjs-public-runtime-variables/
-{
-    # Infer NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED if it is not set
-    if [ -z "$NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED" ] && [ ! -z "$SOURCEBOT_TELEMETRY_DISABLED" ]; then
-        export NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED="$SOURCEBOT_TELEMETRY_DISABLED"
-    fi
-
-    # Infer NEXT_PUBLIC_SOURCEBOT_VERSION if it is not set
-    if [ -z "$NEXT_PUBLIC_SOURCEBOT_VERSION" ] && [ ! -z "$SOURCEBOT_VERSION" ]; then
-        export NEXT_PUBLIC_SOURCEBOT_VERSION="$SOURCEBOT_VERSION"
-    fi
-
-    # Infer NEXT_PUBLIC_PUBLIC_SEARCH_DEMO if it is not set
-    if [ -z "$NEXT_PUBLIC_PUBLIC_SEARCH_DEMO" ] && [ ! -z "$PUBLIC_SEARCH_DEMO" ]; then
-        export NEXT_PUBLIC_PUBLIC_SEARCH_DEMO="$PUBLIC_SEARCH_DEMO"
-    fi
-
-    # Always infer NEXT_PUBLIC_POSTHOG_PAPIK
-    export NEXT_PUBLIC_POSTHOG_PAPIK="$POSTHOG_PAPIK"
-
-    # Always infer NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY
-    export NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY="$STRIPE_PUBLISHABLE_KEY"
-
-    # Always infer NEXT_PUBLIC_SENTRY_ENVIRONMENT
-    export NEXT_PUBLIC_SENTRY_ENVIRONMENT="$SENTRY_ENVIRONMENT"
-
-    # Always infer NEXT_PUBLIC_SENTRY_WEBAPP_DSN
-    export NEXT_PUBLIC_SENTRY_WEBAPP_DSN="$SENTRY_WEBAPP_DSN"
-
-    # Iterate over all .js files in .next & public, making substitutions for the `BAKED_` sentinal values
-    # with their actual desired runtime value.
-    find /app/packages/web/public /app/packages/web/.next -type f -name "*.js" |
-    while read file; do
-        sed -i "s|BAKED_NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED|${NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED}|g" "$file"
-        sed -i "s|BAKED_NEXT_PUBLIC_SOURCEBOT_VERSION|${NEXT_PUBLIC_SOURCEBOT_VERSION}|g" "$file"
-        sed -i "s|BAKED_NEXT_PUBLIC_POSTHOG_PAPIK|${NEXT_PUBLIC_POSTHOG_PAPIK}|g" "$file"
-        sed -i "s|BAKED_NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY|${NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY}|g" "$file"
-        sed -i "s|BAKED_NEXT_PUBLIC_SENTRY_ENVIRONMENT|${NEXT_PUBLIC_SENTRY_ENVIRONMENT}|g" "$file"
-        sed -i "s|BAKED_NEXT_PUBLIC_SENTRY_WEBAPP_DSN|${NEXT_PUBLIC_SENTRY_WEBAPP_DSN}|g" "$file"
-        sed -i "s|BAKED_NEXT_PUBLIC_PUBLIC_SEARCH_DEMO|${NEXT_PUBLIC_PUBLIC_SEARCH_DEMO}|g" "$file"
-    done
-}
-
-# @nocheckin: This was interfering with the the `matcher` regex in middleware.ts,
-# causing regular expressions parsing errors when making a request. It's unclear
-# why exactly this was happening, but it's likely due to a bad replacement happening
-# in the `sed` command.
-#
-# # Update specifically NEXT_PUBLIC_DOMAIN_SUB_PATH w/o requiring a rebuild.
-# # Ultimately, the DOMAIN_SUB_PATH sets the `basePath` param in the next.config.mjs.
-# # Similar to above, we pass in a `BAKED_` sentinal value into next.config.mjs at build
-# # time. Unlike above, the `basePath` configuration is set in files other than just javascript
-# # code (e.g., manifest files, css files, etc.), so this section has subtle differences.
-# #
-# # @see: https://nextjs.org/docs/app/api-reference/next-config-js/basePath
-# # @see: https://phase.dev/blog/nextjs-public-runtime-variables/
-# {
-#     if [ ! -z "$DOMAIN_SUB_PATH" ]; then
-#         # If the sub-path is "/", this creates problems with certain replacements. For example:
-#         # /BAKED_NEXT_PUBLIC_DOMAIN_SUB_PATH/_next/image -> //_next/image (notice the double slash...)
-#         # To get around this, we default to an empty sub-path, which is the default when no sub-path is defined.
-#         if [ "$DOMAIN_SUB_PATH" = "/" ]; then
-#             DOMAIN_SUB_PATH=""
-
-#         # Otherwise, we need to ensure that the sub-path starts with a slash, since this is a requirement
-#         # for the basePath property. For example, assume DOMAIN_SUB_PATH=/bot, then:
-#         # /BAKED_NEXT_PUBLIC_DOMAIN_SUB_PATH/_next/image -> /bot/_next/image
-#         elif [[ ! "$DOMAIN_SUB_PATH" =~ ^/ ]]; then
-#             DOMAIN_SUB_PATH="/$DOMAIN_SUB_PATH"
-#         fi
-#     fi
-
-#     if [ ! -z "$DOMAIN_SUB_PATH" ]; then
-#         echo -e "\e[34m[Info] DOMAIN_SUB_PATH was set to "$DOMAIN_SUB_PATH". Overriding default path.\e[0m"
-#     fi
-
-#     # Always set NEXT_PUBLIC_DOMAIN_SUB_PATH to DOMAIN_SUB_PATH (even if it is empty!!)
-#     export NEXT_PUBLIC_DOMAIN_SUB_PATH="$DOMAIN_SUB_PATH"
-
-#     # Iterate over _all_ files in the web directory, making substitutions for the `BAKED_` sentinal values
-#     # with their actual desired runtime value.
-#     find /app/packages/web -type f |
-#     while read file; do
-#         # @note: the leading "/" is required here as it is included at build time. See Dockerfile.
-#         sed -i "s|/BAKED_NEXT_PUBLIC_DOMAIN_SUB_PATH|${NEXT_PUBLIC_DOMAIN_SUB_PATH}|g" "$file"
-#     done
-# }
-
-# Upload sourcemaps to Sentry
-# @nocheckin
-su -c "sentry-cli login --auth-token $SENTRY_AUTH_TOKEN"
-su -c "sentry-cli sourcemaps inject --org sourcebot --project backend /app/packages/backend/dist"
-su -c "sentry-cli sourcemaps upload --org sourcebot --project backend /app/packages/backend/dist"
-
-
 # Start the database and wait for it to be ready before starting any other service
 if [ "$DATABASE_URL" = "postgresql://postgres@localhost:5432/sourcebot" ]; then
     su postgres -c "postgres -D $DB_DATA_DIR" &

packages/backend/.env

@@ -32,6 +32,7 @@
         "@sourcebot/db": "^0.1.0",
         "@sourcebot/error": "^0.1.0",
         "@sourcebot/schemas": "^0.1.0",
+        "@t3-oss/env-core": "^0.12.0",
         "@types/express": "^5.0.0",
         "argparse": "^2.0.1",
         "bullmq": "^5.34.10",
@@ -47,6 +48,7 @@
         "prom-client": "^15.1.3",
         "simple-git": "^3.27.0",
         "strip-json-comments": "^5.0.1",
-        "winston": "^3.15.0"
+        "winston": "^3.15.0",
+        "zod": "^3.24.2"
     }
 }

packages/backend/package.json

@@ -0,0 +1,41 @@
+import { createEnv } from "@t3-oss/env-core";
+import { z } from "zod";
+import dotenv from 'dotenv';
+
+dotenv.config({
+	path: './.env',
+});
+
+dotenv.config({
+	path: './.env.local',
+	override: true
+});
+
+export const env = createEnv({
+    server: {
+        SOURCEBOT_ENCRYPTION_KEY: z.string(),
+        SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
+        SOURCEBOT_TELEMETRY_DISABLED: z.enum(["true", "false"]).default("false"),
+        SOURCEBOT_INSTALL_ID: z.string().default("unknown"),
+        SOURCEBOT_VERSION: z.string().default("unknown"),
+
+        POSTHOG_PAPIK: z.string().optional(),
+        POSTHOG_HOST: z.string().url().default('https://us.i.posthog.com'),
+
+        FALLBACK_GITHUB_TOKEN: z.string().optional(),
+        FALLBACK_GITLAB_TOKEN: z.string().optional(),
+        FALLBACK_GITEA_TOKEN: z.string().optional(),
+
+        REDIS_URL: z.string().url().optional().default("redis://localhost:6379"),
+
+        SENTRY_BACKEND_DSN: z.string().optional(),
+        SENTRY_ENVIRONMENT: z.string().optional(),
+
+        LOGTAIL_TOKEN: z.string().optional(),
+        LOGTAIL_HOST: z.string().url().optional(),
+    },
+    runtimeEnv: process.env,
+    emptyStringAsUndefined: true,
+    // Skip environment variable validation in Docker builds.
+    skipValidation: process.env.DOCKER_BUILD === "1",
+});