From c035f6cc780db4af399fa9961bfe51bee113f1f8 Mon Sep 17 00:00:00 2001 From: Joosep Alviste Date: Tue, 6 Feb 2024 21:00:32 +0200 Subject: [PATCH] refactor: move jobs to a separate project Related to #83 --- apps/api/bin/build.ts | 2 +- apps/api/package.json | 8 +- apps/api/src/config.ts | 6 -- apps/api/src/lib/bullBoard.ts | 18 ---- apps/jobs/.eslintrc.json | 4 + apps/jobs/.prettierrc | 1 + apps/jobs/Dockerfile | 36 +++++++ apps/jobs/bin/build.ts | 9 ++ apps/jobs/package.json | 32 ++++++ apps/{api => jobs}/src/lib/bullMq.ts | 2 +- apps/jobs/src/lib/config.ts | 8 ++ apps/jobs/src/lib/db.ts | 5 + apps/jobs/src/lib/logger.ts | 3 + .../{api/src/mainJobs.ts => jobs/src/main.ts} | 3 +- apps/jobs/src/seriesSync.job.ts | 26 +++++ apps/jobs/tsconfig.json | 7 ++ docker-compose.production.yml | 6 +- package.json | 4 +- pnpm-lock.yaml | 97 ++++++++----------- 19 files changed, 179 insertions(+), 98 deletions(-) delete mode 100644 apps/api/src/lib/bullBoard.ts create mode 100644 apps/jobs/.eslintrc.json create mode 100644 apps/jobs/.prettierrc create mode 100644 apps/jobs/Dockerfile create mode 100644 apps/jobs/bin/build.ts create mode 100644 apps/jobs/package.json rename apps/{api => jobs}/src/lib/bullMq.ts (94%) create mode 100644 apps/jobs/src/lib/config.ts create mode 100644 apps/jobs/src/lib/db.ts create mode 100644 apps/jobs/src/lib/logger.ts rename apps/{api/src/mainJobs.ts => jobs/src/main.ts} (87%) create mode 100644 apps/jobs/src/seriesSync.job.ts create mode 100644 apps/jobs/tsconfig.json diff --git a/apps/api/bin/build.ts b/apps/api/bin/build.ts index 79104e27..16a21d6c 100644 --- a/apps/api/bin/build.ts +++ b/apps/api/bin/build.ts @@ -10,7 +10,7 @@ const dirname = fileURLToPath(new URL('.', import.meta.url)) await buildEsbuild({ packageJson: pkg, - entryPoints: ['src/main.ts', 'src/mainJobs.ts', 'bin/migrate.ts'], + entryPoints: ['src/main.ts', 'bin/migrate.ts'], tsconfig: 'tsconfig.build.json', external: ['pg-native'], }) diff --git a/apps/api/package.json b/apps/api/package.json index 0652e910..79ae355d 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -4,8 +4,7 @@ "type": "module", "sideEffects": [ "./src/**/*.schema.ts", - "./src/lib/initSentry.ts", - "./src/**/*.job.ts" + "./src/lib/initSentry.ts" ], "scripts": { "build": "rimraf dist && NODE_ENV=production tsx bin/build.ts", @@ -21,8 +20,6 @@ "migrate:test": "dotenv -e ../../.env.test tsx bin/migrate.ts", "start": "NODE_ENV=development tsx watch --clear-screen=false src/main.ts", "start:e2e": "NODE_ENV=test pnpm start:prod", - "start:jobs": "NODE_ENV=development tsx watch --clear-screen=false src/mainJobs.ts", - "start:jobs:prod": "NODE_ENV=production node dist/src/mainJobs.js", "start:prod": "NODE_ENV=production node dist/src/main.js", "test": "dotenv -e ../../.env.test vitest run", "test:coverage": "dotenv -e ../../.env.test vitest run -- --coverage", @@ -30,8 +27,6 @@ "tsc": "tsc --noEmit" }, "dependencies": { - "@bull-board/api": "^5.6.0", - "@bull-board/fastify": "^5.6.0", "@envelop/sentry": "^6.0.0", "@escape.tech/graphql-armor": "^2.2.0", "@fastify/cookie": "^8.3.0", @@ -45,7 +40,6 @@ "@serieslist/feature-series-sync": "workspace:*", "@serieslist/feature-tmdb": "workspace:*", "bcryptjs": "^2.4.3", - "bullmq": "^4.1.0", "dataloader": "^2.2.2", "date-fns": "^2.30.0", "dotenv": "^16.0.3", diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index ff888b6b..6060e6ca 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -23,12 +23,6 @@ export const config = { secretToken: process.env.SECRET_TOKEN!, - redis: { - host: process.env.REDIS_HOST!, - port: parseInt(process.env.REDIS_PORT!), - password: process.env.REDIS_PASSWORD!, - }, - webapp: { host: process.env.APP_HOST!, url: process.env.APP_URL!, diff --git a/apps/api/src/lib/bullBoard.ts b/apps/api/src/lib/bullBoard.ts deleted file mode 100644 index 22b89039..00000000 --- a/apps/api/src/lib/bullBoard.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { createBullBoard as createBullBoardBase } from '@bull-board/api' -import { BullMQAdapter } from '@bull-board/api/bullMQAdapter' -import { FastifyAdapter } from '@bull-board/fastify' - -import { seriesSyncQueue } from '#/features/series/jobs' - -export const createBullBoard = () => { - const bullBoardServerAdapter = new FastifyAdapter() - createBullBoardBase({ - queues: [new BullMQAdapter(seriesSyncQueue)], - serverAdapter: bullBoardServerAdapter, - }) - bullBoardServerAdapter.setBasePath('/bullmq') - - return { - bullBoardServerAdapter, - } -} diff --git a/apps/jobs/.eslintrc.json b/apps/jobs/.eslintrc.json new file mode 100644 index 00000000..1e861a8f --- /dev/null +++ b/apps/jobs/.eslintrc.json @@ -0,0 +1,4 @@ +{ + "extends": "@serieslist/eslint-config-base", + "root": true +} diff --git a/apps/jobs/.prettierrc b/apps/jobs/.prettierrc new file mode 100644 index 00000000..418db572 --- /dev/null +++ b/apps/jobs/.prettierrc @@ -0,0 +1 @@ +"@serieslist/prettier-config" diff --git a/apps/jobs/Dockerfile b/apps/jobs/Dockerfile new file mode 100644 index 00000000..9f436630 --- /dev/null +++ b/apps/jobs/Dockerfile @@ -0,0 +1,36 @@ +FROM node:20-alpine AS base + +RUN npm i -g pnpm@8 + + +FROM base AS dependencies + +WORKDIR /app + +# Install dependencies for compiling node-gyp (since there is no pre-built +# binary for Alpine) +# https://stackoverflow.com/a/59538284/7044732 +RUN apk add g++ make python3 + +COPY . . + +RUN pnpm -F api --prod deploy pruned + + +FROM dependencies AS build + +WORKDIR /app + +RUN rm -rf node_modules && rm -rf pruned && pnpm install +RUN pnpm exec nx build @serieslist/api + + +FROM base AS production + +WORKDIR /app + +COPY --from=build /app/apps/api/dist dist +COPY --from=build /app/apps/api/package.json . +COPY --from=dependencies /app/pruned/node_modules node_modules + +CMD [ "pnpm", "start:prod" ] diff --git a/apps/jobs/bin/build.ts b/apps/jobs/bin/build.ts new file mode 100644 index 00000000..b9d2e23e --- /dev/null +++ b/apps/jobs/bin/build.ts @@ -0,0 +1,9 @@ +import { buildEsbuild } from '@serieslist/core-esbuild' + +import pkg from '../package.json' + +await buildEsbuild({ + packageJson: pkg, + entryPoints: ['src/main.ts'], + external: ['pg-native'], +}) diff --git a/apps/jobs/package.json b/apps/jobs/package.json new file mode 100644 index 00000000..cc2202ff --- /dev/null +++ b/apps/jobs/package.json @@ -0,0 +1,32 @@ +{ + "name": "@serieslist/jobs", + "version": "1.0.0", + "type": "module", + "sideEffects": [ + "src/**/*.job.ts" + ], + "scripts": { + "build": "rimraf dist && NODE_ENV=production tsx bin/build.ts", + "build:docker": "(cd ../.. && docker buildx build --push --cache-to type=gha,mode=max,scope=webapp --cache-from type=gha,scope=webapp -t ghcr.io/joosepalviste/serieslist-jobs:latest -f apps/api/Dockerfile --target production .)", + "lint": "eslint .", + "lint:fix": "eslint --fix .", + "start": "NODE_ENV=development tsx watch --clear-screen=false src/main.ts", + "start:prod": "NODE_ENV=production node dist/main.js", + "tsc": "tsc --noEmit" + }, + "dependencies": { + "@serieslist/core-db": "workspace:*", + "@serieslist/core-logger": "workspace:^", + "@serieslist/feature-series-sync": "workspace:^", + "bullmq": "^4.1.0" + }, + "devDependencies": { + "@serieslist/core-esbuild": "workspace:*", + "@serieslist/eslint-config-base": "workspace:*", + "@serieslist/prettier-config": "workspace:*", + "@serieslist/typescript-config-base": "workspace:*", + "rimraf": "^4.4.0", + "tsx": "^4.7.0", + "typescript": "^5.3.3" + } +} diff --git a/apps/api/src/lib/bullMq.ts b/apps/jobs/src/lib/bullMq.ts similarity index 94% rename from apps/api/src/lib/bullMq.ts rename to apps/jobs/src/lib/bullMq.ts index baf6d003..ebe4b22b 100644 --- a/apps/api/src/lib/bullMq.ts +++ b/apps/jobs/src/lib/bullMq.ts @@ -7,7 +7,7 @@ import { type QueueOptions, } from 'bullmq' -import { config } from '#/config' +import { config } from './config' const redisConnection: RedisOptions = { host: config.redis.host, diff --git a/apps/jobs/src/lib/config.ts b/apps/jobs/src/lib/config.ts new file mode 100644 index 00000000..09c0c56a --- /dev/null +++ b/apps/jobs/src/lib/config.ts @@ -0,0 +1,8 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +export const config = { + redis: { + host: process.env.REDIS_HOST!, + port: parseInt(process.env.REDIS_PORT!), + password: process.env.REDIS_PASSWORD!, + }, +} diff --git a/apps/jobs/src/lib/db.ts b/apps/jobs/src/lib/db.ts new file mode 100644 index 00000000..42c7e6e0 --- /dev/null +++ b/apps/jobs/src/lib/db.ts @@ -0,0 +1,5 @@ +import { createDbConnection } from '@serieslist/core-db' + +import { log } from './logger' + +export const { db } = await createDbConnection({ logger: log }) diff --git a/apps/jobs/src/lib/logger.ts b/apps/jobs/src/lib/logger.ts new file mode 100644 index 00000000..e30fd01f --- /dev/null +++ b/apps/jobs/src/lib/logger.ts @@ -0,0 +1,3 @@ +import { createLogger } from '@serieslist/core-logger' + +export const log = createLogger({ name: 'jobs' }) diff --git a/apps/api/src/mainJobs.ts b/apps/jobs/src/main.ts similarity index 87% rename from apps/api/src/mainJobs.ts rename to apps/jobs/src/main.ts index 059b0c09..241cd850 100644 --- a/apps/api/src/mainJobs.ts +++ b/apps/jobs/src/main.ts @@ -1,6 +1,5 @@ -import { seriesSyncWorker } from '#/features/series/jobs' - import { log } from './lib/logger' +import { seriesSyncWorker } from './seriesSync.job' /** * The job workers are started in a separate container in order to avoid diff --git a/apps/jobs/src/seriesSync.job.ts b/apps/jobs/src/seriesSync.job.ts new file mode 100644 index 00000000..3c36f82a --- /dev/null +++ b/apps/jobs/src/seriesSync.job.ts @@ -0,0 +1,26 @@ +import { reSyncSeries } from '@serieslist/feature-series-sync' + +import { createQueue, createWorker } from './lib/bullMq' +import { db } from './lib/db' + +export const seriesSyncQueue = createQueue('seriesSync') + +const SERIES_SYNC_JOB = 'seriesSync' + +await seriesSyncQueue.add( + SERIES_SYNC_JOB, + {}, + { + repeat: { + pattern: '* * * * *', + }, + }, +) + +export const seriesSyncWorker = createWorker( + SERIES_SYNC_JOB, + async () => { + await reSyncSeries({ ctx: { db } }) + }, + { autorun: false }, +) diff --git a/apps/jobs/tsconfig.json b/apps/jobs/tsconfig.json new file mode 100644 index 00000000..ca567fa1 --- /dev/null +++ b/apps/jobs/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@serieslist/typescript-config-base", + "compilerOptions": { + "baseUrl": ".", + "outDir": "dist" + } +} diff --git a/docker-compose.production.yml b/docker-compose.production.yml index c44d892a..2d4d9f2c 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -20,9 +20,9 @@ services: caddy: api.serieslist.joosep.xyz caddy.reverse_proxy: "{{upstreams ${API_PORT}}}" - api_jobs: - image: ghcr.io/joosepalviste/serieslist-api:latest - command: pnpm start:jobs:prod + jobs: + image: ghcr.io/joosepalviste/serieslist-jobs:latest + command: pnpm start:prod env_file: - .env - .env.docker diff --git a/package.json b/package.json index d374acf2..50c9fbb4 100644 --- a/package.json +++ b/package.json @@ -23,9 +23,9 @@ "migration": "nx run @serieslist/db:migration", "postinstall": "cp -n .env.example .env || exit 0", "quality": "nx run-many -t lint,tsc,test:coverage,build --verbose", - "start": "nx run-many -t start,start:jobs", + "start": "nx run-many -t start", "start:e2e": "nx run-many -t start:e2e", - "start:prod": "nx run-many -t start:prod,start:jobs:prod --output-style stream", + "start:prod": "nx run-many -t start:prod --output-style stream", "test": "nx run-many -t test", "test:coverage": "nx run-many -t test:coverage --verbose", "test:e2e": "nx test:e2e @serieslist/e2e", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3c8e839..be793266 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,12 +26,6 @@ importers: apps/api: dependencies: - '@bull-board/api': - specifier: ^5.6.0 - version: 5.6.0(@bull-board/ui@5.6.0) - '@bull-board/fastify': - specifier: ^5.6.0 - version: 5.6.0 '@envelop/sentry': specifier: ^6.0.0 version: 6.0.0(@envelop/core@4.0.0)(@sentry/node@7.56.0)(graphql@16.8.1) @@ -71,9 +65,6 @@ importers: bcryptjs: specifier: ^2.4.3 version: 2.4.3 - bullmq: - specifier: ^4.1.0 - version: 4.1.0 dataloader: specifier: ^2.2.2 version: 2.2.2 @@ -223,6 +214,43 @@ importers: specifier: ^5.3.3 version: 5.3.3 + apps/jobs: + dependencies: + '@serieslist/core-db': + specifier: workspace:* + version: link:../../packages/core-db + '@serieslist/core-logger': + specifier: workspace:^ + version: link:../../packages/core-logger + '@serieslist/feature-series-sync': + specifier: workspace:^ + version: link:../../packages/feature-series-sync + bullmq: + specifier: ^4.1.0 + version: 4.1.0 + devDependencies: + '@serieslist/core-esbuild': + specifier: workspace:* + version: link:../../packages/core-esbuild + '@serieslist/eslint-config-base': + specifier: workspace:* + version: link:../../packages/eslint-config-base + '@serieslist/prettier-config': + specifier: workspace:* + version: link:../../packages/prettier-config + '@serieslist/typescript-config-base': + specifier: workspace:* + version: link:../../packages/typescript-config-base + rimraf: + specifier: ^4.4.0 + version: 4.4.0 + tsx: + specifier: ^4.7.0 + version: 4.7.0 + typescript: + specifier: ^5.3.3 + version: 5.3.3 + apps/tmdbMockServer: dependencies: '@serieslist/core-logger': @@ -2705,31 +2733,6 @@ packages: '@brillout/import': 0.2.3 dev: false - /@bull-board/api@5.6.0(@bull-board/ui@5.6.0): - resolution: {integrity: sha512-a2O15p5oEm+/E/0I2l2nE2NKK0dkgNNTaamu+0gGyfUxWoCS3fCGX6LLEyl3jgOz0IC3GKRnwtVgbZFzk42sGQ==} - peerDependencies: - '@bull-board/ui': 5.6.0 - dependencies: - '@bull-board/ui': 5.6.0 - redis-info: 3.1.0 - dev: false - - /@bull-board/fastify@5.6.0: - resolution: {integrity: sha512-JIGSrNxRko2cc4KTcU1Pp1iROKuAZGi2aLCx87nlgjmdtK51qbBfpDCsNEEc4ZspdddBKvaTUu67/3/n+3J67w==} - dependencies: - '@bull-board/api': 5.6.0(@bull-board/ui@5.6.0) - '@bull-board/ui': 5.6.0 - '@fastify/static': 6.10.2 - '@fastify/view': 7.4.1 - ejs: 3.1.9 - dev: false - - /@bull-board/ui@5.6.0: - resolution: {integrity: sha512-mc9T+kijDX5ZJMJCzeKPk9uLfOtcuefdDLPqWi961EiiNKfkDex+Gh41DAfcyrNjYsAkBsphvgBgcvundHPGIw==} - dependencies: - '@bull-board/api': 5.6.0(@bull-board/ui@5.6.0) - dev: false - /@cspotcode/source-map-support@0.8.1: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -3802,13 +3805,6 @@ packages: readable-stream: 4.4.2 dev: false - /@fastify/view@7.4.1: - resolution: {integrity: sha512-ahmRmSbNVM8bIoz0BAFnY0jNigom+xbPQ9Q1ZjmNOtGVVT3nYXCxw2OMkTr9iXwrJ4Le3EtWDHlFkZ2fCQ2hJA==} - dependencies: - fastify-plugin: 4.5.0 - hashlru: 2.3.0 - dev: false - /@floating-ui/core@1.3.1: resolution: {integrity: sha512-Bu+AMaXNjrpjh41znzHqaz3r2Nr8hHuHZT6V2LBKMhyMl0FgKA62PNYbqnfgmzOhoWZj70Zecisbo4H1rotP5g==} dev: false @@ -7903,7 +7899,7 @@ packages: std-env: 3.7.0 test-exclude: 6.0.0 v8-to-istanbul: 9.2.0 - vitest: 1.1.1(@types/node@18.16.9)(jsdom@22.1.0) + vitest: 1.1.1(@types/node@18.15.3) transitivePeerDependencies: - supports-color dev: true @@ -11360,10 +11356,6 @@ packages: dependencies: function-bind: 1.1.2 - /hashlru@2.3.0: - resolution: {integrity: sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A==} - dev: false - /hasown@2.0.0: resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==} engines: {node: '>= 0.4'} @@ -12716,11 +12708,6 @@ packages: dependencies: tslib: 2.6.0 - /lru-cache@10.0.0: - resolution: {integrity: sha512-svTf/fzsKHffP42sujkO/Rjs37BCIsQVRCeNYIm9WN8rgT7ffoUnRtZCqU+6BqcSBdv8gwJeTz8knJpgACeQMw==} - engines: {node: 14 || >=16.14} - dev: true - /lru-cache@10.1.0: resolution: {integrity: sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==} engines: {node: 14 || >=16.14} @@ -13657,7 +13644,7 @@ packages: resolution: {integrity: sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==} engines: {node: '>=16 || 14 >=14.17'} dependencies: - lru-cache: 10.0.0 + lru-cache: 10.1.0 minipass: 7.0.1 dev: true @@ -14344,12 +14331,6 @@ packages: engines: {node: '>=4'} dev: false - /redis-info@3.1.0: - resolution: {integrity: sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==} - dependencies: - lodash: 4.17.21 - dev: false - /redis-parser@3.0.0: resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} engines: {node: '>=4'}