feat: Load plugins & routes from filesystem

Use `fastify-autoload` for plugins and routes rather than
using `configure` and `routesDir` options (now deprecated).
This allows passing full autoload configurations.

Add `cleanupOnExit` to automate closing connections to
external services when the server is closing.

Reorder plugins to:
- Be able to use loaded plugins decorators in health check
- Disable graceful shutdown in tests

Detect plugin loading errors and crash early (rethrow).

Add unit tests & integration test server.

package.json

@@ -28,52 +28,52 @@
     "build:ts": "tsc",
     "build": "run-s build:clean build:ts",
     "ci": "run-s build test",
+    "test:integration": "NODE_ENV=production ts-node ./tests/integration/main.ts",
     "prepare": "husky install"
   },
   "dependencies": {
     "@47ng/check-env": "^2.0.2",
-    "@sentry/node": "^6.15.0",
-    "fastify": "^3.24.0",
+    "@sentry/node": "^6.16.0",
+    "fastify": "^3.24.1",
     "fastify-autoload": "^3.9.0",
     "fastify-graceful-shutdown": "^3.1.0",
     "fastify-plugin": "^3.0.0",
     "fastify-sensible": "^3.1.2",
     "get-port": "^6.0.0",
     "nanoid": "^3.1.30",
     "redact-env": "^0.3.1",
-    "sonic-boom": "^2.3.1",
+    "sonic-boom": "^2.4.1",
     "under-pressure": "^5.8.0"
   },
   "devDependencies": {
     "@commitlint/config-conventional": "^15.0.0",
+    "@swc/cli": "^0.1.52",
+    "@swc/core": "^1.2.118",
+    "@swc/helpers": "^0.3.2",
     "@types/jest": "^27.0.3",
-    "@types/node": "^16.11.9",
+    "@types/node": "^16.11.12",
     "@types/pino": "7.0.5",
     "@types/sonic-boom": "^2.1.1",
     "axios": "^0.24.0",
     "commitlint": "^15.0.0",
-    "husky": "^7.0.0",
-    "jest": "^27.3.1",
+    "husky": "^7.0.4",
+    "jest": "^27.4.3",
     "npm-run-all": "^4.1.5",
+    "regenerator-runtime": "^0.13.9",
     "sentry-testkit": "^3.3.7",
-    "ts-jest": "^27.0.7",
+    "ts-jest": "^27.1.0",
     "ts-node": "^10.4.0",
     "typescript": "^4.5.2",
     "wait-for-expect": "^3.0.2"
   },
-  "nodemon": {
-    "verbose": false,
-    "execMap": {
-      "ts": "ts-node"
-    },
-    "ignore": [
-      "./dist"
-    ]
-  },
   "jest": {
     "verbose": true,
     "preset": "ts-jest/presets/js-with-ts",
-    "testEnvironment": "node"
+    "testEnvironment": "node",
+    "testPathIgnorePatterns": [
+      "/node_modules/",
+      "<rootDir>/tests/integration/"
+    ]
   },
   "prettier": {
     "arrowParens": "avoid",

src/index.ts

@@ -1,18 +1,16 @@
 import checkEnv from '@47ng/check-env'
 import Fastify, { FastifyInstance, FastifyServerOptions } from 'fastify'
-import autoLoad from 'fastify-autoload'
-// @ts-ignore
+import { AutoloadPluginOptions, fastifyAutoload } from 'fastify-autoload'
 import gracefulShutdown from 'fastify-graceful-shutdown'
 import 'fastify-sensible'
 import sensible from 'fastify-sensible'
 import underPressurePlugin from 'under-pressure'
 import { getLoggerOptions, makeReqIdGenerator } from './logger'
-import sentry, { SentryDecoration, SentryOptions } from './sentry'
+import sentry, { SentryOptions } from './sentry'
 
 declare module 'fastify' {
   interface FastifyInstance {
     name?: string
-    sentry: SentryDecoration
   }
 }
 
@@ -47,6 +45,8 @@ export type Options = FastifyServerOptions & {
   redactLogPaths?: string[]
 
   /**
+   * @deprecated - Use `plugins` instead to load plugins from the filesystem.
+   *
    * Add your own plugins in this callback.
    *
    * It's called after most built-in plugins have run,
@@ -71,6 +71,22 @@ export type Options = FastifyServerOptions & {
   sentry?: SentryOptions
 
   /**
+   * Load plugins from the filesystem with `fastify-autoload`.
+   *
+   * Plugins are loaded before routes (see `routes` option).
+   */
+  plugins?: AutoloadPluginOptions
+
+  /**
+   * Load routes from the filesystem with `fastify-autoload`.
+   *
+   * Routes are loaded after plugins (see `plugins` option).
+   */
+  routes?: AutoloadPluginOptions
+
+  /**
+   * @deprecated - Use `routes` instead, with full `fastify-autoload` options.
+   *
    * Path to a directory where to load routes.
    *
    * This directory will be walked recursively and any file encountered
@@ -81,6 +97,13 @@ export type Options = FastifyServerOptions & {
    */
   routesDir?: string | false
 
+  /**
+   * Run cleanup tasks before exiting.
+   *
+   * Eg: disconnecting backing services, closing files...
+   */
+  cleanupOnExit?: (server: FastifyInstance) => Promise<void>
+
   /**
    * Print routes after server has loaded
    *
@@ -97,16 +120,15 @@ export type Options = FastifyServerOptions & {
 
 export function createServer(
   options: Options = {
-    routesDir: false,
-    printRoutes: 'auto'
+    printRoutes: 'auto',
+    routesDir: false
   }
 ) {
   checkEnv({ required: ['NODE_ENV'] })
 
   const server = Fastify({
     logger: getLoggerOptions(options),
-    // todo: Fix type when switching to Fastify 3.x
-    genReqId: makeReqIdGenerator() as any,
+    genReqId: makeReqIdGenerator(),
     trustProxy: process.env.TRUSTED_PROXY_IPS,
     ...options
   })
@@ -117,71 +139,118 @@ export function createServer(
   server.register(sensible)
   server.register(sentry, options.sentry as any)
 
-  // Disable graceful shutdown if signal listeners are already in use
-  // (eg: using Clinic.js or other kinds of wrapping utilities)
-  const gracefulSignals = ['SIGINT', 'SIGTERM'].filter(
-    signal => process.listenerCount(signal) > 0
-  )
-  if (gracefulSignals.length === 0) {
-    server.register(gracefulShutdown)
-  } else if (process.env.NODE_ENV === 'production') {
-    server.log.warn({
-      plugin: 'fastify-graceful-shutdown',
-      msg: 'Automatic graceful shutdown is disabled',
-      reason: 'Some signal handlers were already registered',
-      signals: gracefulSignals
-    })
-  }
+  try {
+    if (options.plugins) {
+      server.register(fastifyAutoload, options.plugins)
+    }
+    if (options.configure) {
+      if (process.env.NODE_ENV === 'development') {
+        console.warn(
+          '[fastify-micro] Option `configure` is deprecated. Use `plugins` instead with full fastify-autoload options.'
+        )
+      }
+      options.configure(server)
+    }
 
-  if (options.configure) {
-    options.configure(server)
-  }
+    // Registered after plugins to let the health check callback
+    // monitor external services' health.
+    if (
+      process.env.FASTIFY_MICRO_DISABLE_SERVICE_HEALTH_MONITORING !== 'true'
+    ) {
+      const underPressureOptions = options.underPressure || {}
+      server
+        .after(error => {
+          if (error) {
+            throw error
+          }
+        })
+        .register(underPressurePlugin, {
+          maxEventLoopDelay: 1000, // 1s
+          // maxHeapUsedBytes: 100 * (1 << 20), // 100 MiB
+          // maxRssBytes: 100 * (1 << 20), // 100 MiB
+          healthCheckInterval: 5000, // 5 seconds
+          exposeStatusRoute: {
+            url: '/_health',
+            routeOpts: {
+              logLevel: 'warn'
+            }
+          },
+          ...underPressureOptions
+        })
+    }
 
-  if (options.routesDir) {
-    server.register(autoLoad, {
-      dir: options.routesDir
-    })
-  }
+    // Disable graceful shutdown if signal listeners are already in use
+    // (eg: using Clinic.js or other kinds of wrapping utilities)
+    const gracefulSignals = ['SIGINT', 'SIGTERM'].filter(
+      signal => process.listenerCount(signal) > 0
+    )
 
-  if (process.env.FASTIFY_MICRO_DISABLE_SERVICE_HEALTH_MONITORING !== 'true') {
-    const underPressureOptions = options.underPressure || {}
-    server.register(underPressurePlugin, {
-      maxEventLoopDelay: 1000, // 1s
-      // maxHeapUsedBytes: 100 * (1 << 20), // 100 MiB
-      // maxRssBytes: 100 * (1 << 20), // 100 MiB
-      healthCheckInterval: 5000, // 5 seconds
-      exposeStatusRoute: {
-        url: '/_health',
-        routeOpts: {
-          logLevel: 'warn'
-        }
-      },
-      ...underPressureOptions
-    })
-  }
+    if (gracefulSignals.length === 0 && process.env.NODE_ENV !== 'test') {
+      server.register(gracefulShutdown)
+    } else if (process.env.NODE_ENV === 'production') {
+      server.log.warn({
+        plugin: 'fastify-graceful-shutdown',
+        msg: 'Automatic graceful shutdown is disabled',
+        reason: 'Some signal handlers were already registered',
+        signals: gracefulSignals
+      })
+    }
+
+    if (options.routes) {
+      server.register(fastifyAutoload, options.routes)
+    }
+    if (options.routesDir) {
+      if (process.env.NODE_ENV === 'development') {
+        console.warn(
+          '[fastify-micro] Option `routesDir` is deprecated. Use `routes` instead with full fastify-autoload options.'
+        )
+      }
+      server.register(fastifyAutoload, {
+        dir: options.routesDir
+      })
+    }
+
+    if (options.cleanupOnExit) {
+      server.addHook('onClose', options.cleanupOnExit)
+    }
 
-  if (options.printRoutes !== false) {
-    switch (options.printRoutes || 'auto') {
-      default:
-      case 'auto':
-        if (process.env.NODE_ENV === 'development') {
-          server.ready(() => console.info(server.printRoutes()))
-        }
-        break
-      case 'console':
-        server.ready(() => console.info(server.printRoutes()))
-        break
-      case 'logger':
-        server.ready(() =>
+    server.ready(error => {
+      if (error) {
+        // This will let the server crash early
+        // on plugin/routes loading errors.
+        throw error
+      }
+      if (options.printRoutes === false) {
+        return
+      }
+      switch (options.printRoutes || 'auto') {
+        default:
+        case 'auto':
+          if (process.env.NODE_ENV === 'development') {
+            console.info(server.printRoutes())
+          }
+          break
+        case 'console':
+          console.info(server.printRoutes())
+          break
+        case 'logger':
           server.log.info({
             msg: 'Routes loaded',
             routes: server.printRoutes()
           })
-        )
-        break
+          break
+      }
+    })
+  } catch (error) {
+    server.log.fatal(error)
+    if (!server.sentry) {
+      process.exit(1)
     }
+    server.sentry
+      .report(error as any)
+      .catch(error => server.log.fatal(error))
+      .finally(() => process.exit(1))
   }
-
   return server
 }
 
@@ -199,7 +268,16 @@ export async function startServer(
   server: FastifyInstance,
   port: number = parseInt(process.env.PORT || '3000') || 3000
 ) {
-  await server.ready()
+  await server.ready().then(
+    () => {
+      server.log.debug('Starting server')
+    },
+    error => {
+      if (error) {
+        throw error
+      }
+    }
+  )
   return await new Promise(resolve => {
     server.listen({ port, host: '0.0.0.0' }, (error, address) => {
       if (error) {

src/logger.ts

@@ -1,7 +1,6 @@
-import crypto from 'crypto'
-import { FastifyLoggerOptions } from 'fastify'
-import { IncomingMessage } from 'http'
+import type { FastifyLoggerOptions, FastifyRequest } from 'fastify'
 import { nanoid } from 'nanoid'
+import crypto from 'node:crypto'
 import pino from 'pino'
 import redactEnv from 'redact-env'
 import SonicBoom from 'sonic-boom'
@@ -75,7 +74,10 @@ export function getLoggerOptions({
                 name === 'content-length'
                   ? parseInt(rest[0], 10)
                   : rest.join(': ')
-              return Object.assign(obj, { [name]: value })
+              return {
+                ...obj,
+                [name]: value
+              }
             } catch {
               return obj
             }
@@ -90,7 +92,7 @@ export function getLoggerOptions({
 }
 
 export const makeReqIdGenerator = (defaultSalt: string = nanoid()) =>
-  function genReqId(req: IncomingMessage): string {
+  function genReqId(req: FastifyRequest): string {
     let ipAddress: string = ''
     const xForwardedFor = req.headers['x-forwarded-for']
     if (xForwardedFor) {

src/sentry.ts

@@ -1,5 +1,5 @@
 import * as Sentry from '@sentry/node'
-import {
+import type {
   FastifyError,
   FastifyInstance,
   FastifyPluginCallback,
@@ -8,6 +8,9 @@ import {
 import fp from 'fastify-plugin'
 
 declare module 'fastify' {
+  interface FastifyInstance {
+    sentry: SentryDecoration
+  }
   interface FastifyRequest {
     sentry: SentryDecoration
   }