refactor: move jobs to a separate project

Related to #83

apps/api/bin/build.ts

@@ -10,7 +10,7 @@ const dirname = fileURLToPath(new URL('.', import.meta.url))
 
 await buildEsbuild({
   packageJson: pkg,
-  entryPoints: ['src/main.ts', 'src/mainJobs.ts', 'bin/migrate.ts'],
+  entryPoints: ['src/main.ts', 'bin/migrate.ts'],
   tsconfig: 'tsconfig.build.json',
   external: ['pg-native'],
 })

apps/api/package.json

@@ -4,8 +4,7 @@
   "type": "module",
   "sideEffects": [
     "./src/**/*.schema.ts",
-    "./src/lib/initSentry.ts",
-    "./src/**/*.job.ts"
+    "./src/lib/initSentry.ts"
   ],
   "scripts": {
     "build": "rimraf dist && NODE_ENV=production tsx bin/build.ts",
@@ -21,17 +20,13 @@
     "migrate:test": "dotenv -e ../../.env.test tsx bin/migrate.ts",
     "start": "NODE_ENV=development tsx watch --clear-screen=false src/main.ts",
     "start:e2e": "NODE_ENV=test pnpm start:prod",
-    "start:jobs": "NODE_ENV=development tsx watch --clear-screen=false src/mainJobs.ts",
-    "start:jobs:prod": "NODE_ENV=production node dist/src/mainJobs.js",
     "start:prod": "NODE_ENV=production node dist/src/main.js",
     "test": "dotenv -e ../../.env.test vitest run",
     "test:coverage": "dotenv -e ../../.env.test vitest run -- --coverage",
     "test:watch": "dotenv -e ../../.env.test vitest",
     "tsc": "tsc --noEmit"
   },
   "dependencies": {
-    "@bull-board/api": "^5.6.0",
-    "@bull-board/fastify": "^5.6.0",
     "@envelop/sentry": "^6.0.0",
     "@escape.tech/graphql-armor": "^2.2.0",
     "@fastify/cookie": "^8.3.0",
@@ -45,7 +40,6 @@
     "@serieslist/feature-series-sync": "workspace:*",
     "@serieslist/feature-tmdb": "workspace:*",
     "bcryptjs": "^2.4.3",
-    "bullmq": "^4.1.0",
     "dataloader": "^2.2.2",
     "date-fns": "^2.30.0",
     "dotenv": "^16.0.3",

apps/api/src/config.ts

@@ -23,12 +23,6 @@ export const config = {
 
   secretToken: process.env.SECRET_TOKEN!,
 
-  redis: {
-    host: process.env.REDIS_HOST!,
-    port: parseInt(process.env.REDIS_PORT!),
-    password: process.env.REDIS_PASSWORD!,
-  },
-
   webapp: {
     host: process.env.APP_HOST!,
     url: process.env.APP_URL!,

apps/jobs/.eslintrc.json

@@ -0,0 +1,4 @@
+{
+  "extends": "@serieslist/eslint-config-base",
+  "root": true
+}

apps/jobs/.prettierrc

@@ -0,0 +1 @@
+"@serieslist/prettier-config"

apps/jobs/Dockerfile

@@ -0,0 +1,36 @@
+FROM node:20-alpine AS base
+
+RUN npm i -g pnpm@8
+
+
+FROM base AS dependencies
+
+WORKDIR /app
+
+# Install dependencies for compiling node-gyp (since there is no pre-built 
+# binary for Alpine)
+# https://stackoverflow.com/a/59538284/7044732
+RUN apk add g++ make python3
+
+COPY . .
+
+RUN pnpm -F api --prod deploy pruned
+
+
+FROM dependencies AS build
+
+WORKDIR /app
+
+RUN rm -rf node_modules && rm -rf pruned && pnpm install
+RUN pnpm exec nx build @serieslist/api
+
+
+FROM base AS production
+
+WORKDIR /app
+
+COPY --from=build /app/apps/api/dist dist
+COPY --from=build /app/apps/api/package.json .
+COPY --from=dependencies /app/pruned/node_modules node_modules
+
+CMD [ "pnpm", "start:prod" ]

apps/jobs/bin/build.ts

@@ -0,0 +1,9 @@
+import { buildEsbuild } from '@serieslist/core-esbuild'
+
+import pkg from '../package.json'
+
+await buildEsbuild({
+  packageJson: pkg,
+  entryPoints: ['src/main.ts'],
+  external: ['pg-native'],
+})

apps/jobs/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@serieslist/jobs",
+  "version": "1.0.0",
+  "type": "module",
+  "sideEffects": [
+    "src/**/*.job.ts"
+  ],
+  "scripts": {
+    "build": "rimraf dist && NODE_ENV=production tsx bin/build.ts",
+    "build:docker": "(cd ../.. && docker buildx build --push --cache-to type=gha,mode=max,scope=webapp --cache-from type=gha,scope=webapp -t ghcr.io/joosepalviste/serieslist-jobs:latest -f apps/api/Dockerfile --target production .)",
+    "lint": "eslint .",
+    "lint:fix": "eslint --fix .",
+    "start": "NODE_ENV=development tsx watch --clear-screen=false src/main.ts",
+    "start:prod": "NODE_ENV=production node dist/main.js",
+    "tsc": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@serieslist/core-db": "workspace:*",
+    "@serieslist/core-logger": "workspace:^",
+    "@serieslist/feature-series-sync": "workspace:^",
+    "bullmq": "^4.1.0"
+  },
+  "devDependencies": {
+    "@serieslist/core-esbuild": "workspace:*",
+    "@serieslist/eslint-config-base": "workspace:*",
+    "@serieslist/prettier-config": "workspace:*",
+    "@serieslist/typescript-config-base": "workspace:*",
+    "rimraf": "^4.4.0",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/api/src/lib/bullMq.ts → apps/jobs/src/lib/bullMq.ts

@@ -7,7 +7,7 @@ import {
   type QueueOptions,
 } from 'bullmq'
 
-import { config } from '#/config'
+import { config } from './config'
 
 const redisConnection: RedisOptions = {
   host: config.redis.host,

apps/jobs/src/lib/config.ts

@@ -0,0 +1,8 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+export const config = {
+  redis: {
+    host: process.env.REDIS_HOST!,
+    port: parseInt(process.env.REDIS_PORT!),
+    password: process.env.REDIS_PASSWORD!,
+  },
+}

apps/jobs/src/lib/db.ts

@@ -0,0 +1,5 @@
+import { createDbConnection } from '@serieslist/core-db'
+
+import { log } from './logger'
+
+export const { db } = await createDbConnection({ logger: log })

apps/jobs/src/lib/logger.ts

@@ -0,0 +1,3 @@
+import { createLogger } from '@serieslist/core-logger'
+
+export const log = createLogger({ name: 'jobs' })

apps/api/src/mainJobs.ts → apps/jobs/src/main.ts

@@ -1,6 +1,5 @@
-import { seriesSyncWorker } from '#/features/series/jobs'
-
 import { log } from './lib/logger'
+import { seriesSyncWorker } from './seriesSync.job'
 
 /**
  * The job workers are started in a separate container in order to avoid

apps/jobs/src/seriesSync.job.ts

@@ -0,0 +1,26 @@
+import { reSyncSeries } from '@serieslist/feature-series-sync'
+
+import { createQueue, createWorker } from './lib/bullMq'
+import { db } from './lib/db'
+
+export const seriesSyncQueue = createQueue('seriesSync')
+
+const SERIES_SYNC_JOB = 'seriesSync'
+
+await seriesSyncQueue.add(
+  SERIES_SYNC_JOB,
+  {},
+  {
+    repeat: {
+      pattern: '* * * * *',
+    },
+  },
+)
+
+export const seriesSyncWorker = createWorker(
+  SERIES_SYNC_JOB,
+  async () => {
+    await reSyncSeries({ ctx: { db } })
+  },
+  { autorun: false },
+)

apps/jobs/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "@serieslist/typescript-config-base",
+  "compilerOptions": {
+    "baseUrl": ".",
+    "outDir": "dist"
+  }
+}

docker-compose.production.yml

@@ -20,9 +20,9 @@ services:
       caddy: api.serieslist.joosep.xyz
       caddy.reverse_proxy: "{{upstreams ${API_PORT}}}"
 
-  api_jobs:
-    image: ghcr.io/joosepalviste/serieslist-api:latest
-    command: pnpm start:jobs:prod
+  jobs:
+    image: ghcr.io/joosepalviste/serieslist-jobs:latest
+    command: pnpm start:prod
     env_file:
       - .env
       - .env.docker

package.json

@@ -23,9 +23,9 @@
     "migration": "nx run @serieslist/db:migration",
     "postinstall": "cp -n .env.example .env || exit 0",
     "quality": "nx run-many -t lint,tsc,test:coverage,build --verbose",
-    "start": "nx run-many -t start,start:jobs",
+    "start": "nx run-many -t start",
     "start:e2e": "nx run-many -t start:e2e",
-    "start:prod": "nx run-many -t start:prod,start:jobs:prod --output-style stream",
+    "start:prod": "nx run-many -t start:prod --output-style stream",
     "test": "nx run-many -t test",
     "test:coverage": "nx run-many -t test:coverage --verbose",
     "test:e2e": "nx test:e2e @serieslist/e2e",