fix(jobs): load the OpenTelemetry setup in every process running app code

packages/api-server-watch/src/serverManager.ts

@@ -1,14 +1,18 @@
 import type { ChildProcess } from 'child_process'
 import { fork } from 'child_process'
-import fs from 'node:fs'
 import { createRequire } from 'node:module'
 import path from 'path'
 
 import ansis from 'ansis'
 import yargs from 'yargs'
 import { hideBin } from 'yargs/helpers'
 
-import { getConfig, getPaths, resolveFile } from '@cedarjs/project-config'
+import {
+  getConfig,
+  getOTelImportArgs,
+  getPaths,
+  resolveFile,
+} from '@cedarjs/project-config'
 
 const require = createRequire(import.meta.url)
 
@@ -32,32 +36,7 @@ export class ServerManager {
 
   private async startApiServer() {
     const forkOpts = {
-      execArgv: process.execArgv,
-    }
-
-    // OpenTelemetry SDK Setup
-    if (getConfig().experimental.opentelemetry.enabled) {
-      // We expect the OpenTelemetry SDK setup file to be in a specific location
-      const opentelemetrySDKScriptPath = path.join(
-        rwjsPaths.api.dist,
-        'opentelemetry.js',
-      )
-      const opentelemetrySDKScriptPathRelative = path.relative(
-        rwjsPaths.base,
-        opentelemetrySDKScriptPath,
-      )
-      console.log(
-        `Setting up OpenTelemetry using the setup file: ${opentelemetrySDKScriptPathRelative}`,
-      )
-      if (fs.existsSync(opentelemetrySDKScriptPath)) {
-        forkOpts.execArgv = forkOpts.execArgv.concat([
-          `--require=${opentelemetrySDKScriptPath}`,
-        ])
-      } else {
-        console.error(
-          `OpenTelemetry setup file does not exist at ${opentelemetrySDKScriptPathRelative}`,
-        )
-      }
+      execArgv: process.execArgv.concat(getOTelImportArgs()),
     }
 
     const debugPort = argv['debug-port']

packages/api-server/src/apiCLIConfigHandler.ts

@@ -1,8 +1,10 @@
 import ansis from 'ansis'
 
-import { coerceRootPath } from '@cedarjs/fastify-web'
+import {
+  getOTelSetupFileUrl,
+  isCLITelemetryActive,
+} from '@cedarjs/project-config'
 
-import { createServer } from './createServer.js'
 import { apiDistServerFileExists, runApiDistServerFile } from './serverFile.js'
 import type { APIParsedOptions } from './types.js'
 
@@ -15,6 +17,25 @@ export async function handler(options: APIParsedOptions = {}) {
     return
   }
 
+  // The OpenTelemetry SDK setup must be imported before Fastify, Prisma and
+  // the app's own modules are loaded, or the instrumentations can't patch
+  // them. When this runs in its own process (`cedarjs-server api`, started
+  // with the `--import` flag) the setup is already loaded, and importing it
+  // again is a no-op. When it runs inside the `cedar` CLI process
+  // (`cedar serve api` without a server file), CLI telemetry may already
+  // have registered a global tracer provider, in which case the app's setup
+  // can't become the global provider and is skipped: its spans would be
+  // misrouted into the CLI's telemetry exporter. Fastify and the server
+  // factory are imported after the setup for the same reason.
+  const otelSetupFileUrl = getOTelSetupFileUrl()
+
+  if (otelSetupFileUrl !== undefined && !isCLITelemetryActive()) {
+    await import(otelSetupFileUrl)
+  }
+
+  const { coerceRootPath } = await import('@cedarjs/fastify-web')
+  const { createServer } = await import('./createServer.js')
+
   const timeStart = Date.now()
   console.log(ansis.dim.italic('Starting API Server...'))
 

packages/api-server/src/bin.ts

@@ -15,12 +15,10 @@ import {
   description as apiDescription,
   builder as apiBuilder,
 } from './apiCLIConfig.js'
-import { handler as apiHandler } from './apiCLIConfigHandler.js'
 import {
   description as bothDescription,
   builder as bothBuilder,
 } from './bothCLIConfig.js'
-import { handler as bothHandler } from './bothCLIConfigHandler.js'
 
 if (!process.env.CEDAR_ENV_FILES_LOADED) {
   config({
@@ -44,14 +42,23 @@ yargs(hideBin(process.argv))
     bothDescription,
     // @ts-expect-error The yargs types seem wrong; it's ok for builder to be a function
     bothBuilder,
-    bothHandler,
+    // The API handlers import Fastify and the app's own modules, so they are
+    // loaded lazily: the OpenTelemetry setup they import must be able to run
+    // first through the preload flags placed on this process.
+    async (argv: never) => {
+      const { handler } = await import('./bothCLIConfigHandler.js')
+      await handler(argv)
+    },
   )
   .command(
     'api',
     apiDescription,
     // @ts-expect-error The yargs types seem wrong; it's ok for builder to be a function
     apiBuilder,
-    apiHandler,
+    async (argv: never) => {
+      const { handler } = await import('./apiCLIConfigHandler.js')
+      await handler(argv)
+    },
   )
   .command(
     'web',

packages/api-server/src/bothCLIConfigHandler.ts

@@ -1,10 +1,11 @@
 import ansis from 'ansis'
 
-import { redwoodFastifyWeb, coerceRootPath } from '@cedarjs/fastify-web'
+import {
+  getOTelSetupFileUrl,
+  isCLITelemetryActive,
+} from '@cedarjs/project-config'
 
 import { getWebHost, getWebPort, getAPIHost, getAPIPort } from './cliHelpers.js'
-import { createServer as createApiServer } from './createServer.js'
-import { createFastifyInstance } from './fastify.js'
 import { apiDistServerFileExists } from './serverFile.js'
 import type { BothParsedOptions } from './types.js'
 
@@ -29,6 +30,27 @@ export async function handler(options: BothParsedOptions) {
     )
   }
 
+  // The OpenTelemetry SDK setup must be imported before Fastify, Prisma and
+  // the app's own modules are loaded, or the instrumentations can't patch
+  // them. When this runs in its own process (`cedarjs-server`, started with
+  // the `--import` flag) the setup is already loaded, and importing it again
+  // is a no-op. When it runs inside the `cedar` CLI process (`cedar serve`
+  // without a server file), CLI telemetry may already have registered a
+  // global tracer provider, in which case the app's setup can't become the
+  // global provider and is skipped: its spans would be misrouted into the
+  // CLI's telemetry exporter. Fastify and the server factories are imported
+  // after the setup for the same reason.
+  const otelSetupFileUrl = getOTelSetupFileUrl()
+
+  if (otelSetupFileUrl !== undefined && !isCLITelemetryActive()) {
+    await import(otelSetupFileUrl)
+  }
+
+  const { redwoodFastifyWeb, coerceRootPath } =
+    await import('@cedarjs/fastify-web')
+  const { createServer: createApiServer } = await import('./createServer.js')
+  const { createFastifyInstance } = await import('./fastify.js')
+
   const timeStart = Date.now()
   console.log(ansis.dim.italic('Starting API and Web Servers...'))
 

packages/api-server/src/serverFile.ts

@@ -2,7 +2,7 @@ import { spawn } from 'node:child_process'
 import fs from 'node:fs'
 import path from 'node:path'
 
-import { getPaths } from '@cedarjs/project-config'
+import { getOTelImportArgs, getPaths } from '@cedarjs/project-config'
 
 import type { APIParsedOptions } from './types.js'
 
@@ -44,10 +44,14 @@ export async function runApiDistServerFile(
   }
 
   await new Promise<void>((resolve, reject) => {
-    const child = spawn(process.execPath, [apiDistServerFilePath(), ...args], {
-      cwd: getPaths().api.dist,
-      stdio: 'inherit',
-    })
+    const child = spawn(
+      process.execPath,
+      [...getOTelImportArgs(), apiDistServerFilePath(), ...args],
+      {
+        cwd: getPaths().api.dist,
+        stdio: 'inherit',
+      },
+    )
 
     child.on('error', reject)
 

packages/cli/src/commands/experimental/setupOpentelemetryHandler.ts

@@ -146,6 +146,23 @@ export const handler = async ({
       rendererOptions: { persistentOutput: true },
     },
     addApiPackages(opentelemetryPackages),
+    {
+      title: 'Notice: running under your own process manager...',
+      task: (_ctx: unknown, task: { output: string }) => {
+        task.output = [
+          'The setup file is loaded automatically by `cedar dev`, `cedarjs-server api` and the job workers.',
+          '`cedar serve api` and `cedar serve` without a custom server file run the API in the `cedar` process, where the CLI telemetry provider is already registered, so the setup file is skipped there unless you disable CLI telemetry with `--telemetry false`.',
+          'If you run the API or the job workers under your own process manager, load it yourself:',
+          '',
+          c.info(
+            'node --import ./api/dist/opentelemetry.js path/to/your/server.js',
+          ),
+          '',
+          '(or set the same flag in NODE_OPTIONS)',
+        ].join('\n')
+      },
+      rendererOptions: { persistentOutput: true },
+    },
   ]
 
   const prismaTasks = [

packages/cli/src/commands/serve.ts

@@ -64,6 +64,31 @@ async function startUDServer(
   host: string,
   port: number,
 ): Promise<ReturnType<typeof serveSrvx>> {
+  // The OpenTelemetry SDK setup must be imported before the built Universal
+  // Deploy Fetchable is loaded, or the instrumentations can't patch it. The
+  // `--import` entries `getOTelImportArgs()` returns are for child processes;
+  // here the setup file is imported in-process. This CLI process may already
+  // have the CLI telemetry provider registered globally, in which case the
+  // app's setup can't become the global provider here and is skipped: its
+  // spans would be misrouted into the CLI's telemetry exporter.
+  const { getOTelSetupFileUrl, isCLITelemetryActive } =
+    await import('@cedarjs/project-config')
+  const otelSetupFileUrl = getOTelSetupFileUrl()
+
+  if (otelSetupFileUrl !== undefined) {
+    if (isCLITelemetryActive()) {
+      console.warn(
+        'OpenTelemetry: the CLI telemetry provider is already registered in ' +
+          "this process, so the app's OpenTelemetry setup file cannot be " +
+          'loaded here. Run `cedar serve --telemetry false` to disable CLI ' +
+          "telemetry and let the app's setup take over, or serve the API in " +
+          'a separate process.',
+      )
+    } else {
+      await import(otelSetupFileUrl)
+    }
+  }
+
   const mod = await import(pathToFileURL(entryPath).href)
   const fetchable = mod.default ?? mod
 

packages/cli/src/commands/serveApiHandler.ts

@@ -1,13 +1,18 @@
 import { runBin } from '@cedarjs/cli-helpers/packageManager/exec'
-import { getPaths } from '@cedarjs/project-config'
+import { getOTelImportArgs, getPaths } from '@cedarjs/project-config'
 
 interface ServeApiArgv {
   apiRootPath?: string
   port?: number
 }
 
 export async function apiServerFileHandler(argv: ServeApiArgv) {
-  const args = ['server.js', '--apiRootPath', argv.apiRootPath ?? '/']
+  const args = [
+    ...getOTelImportArgs(),
+    'server.js',
+    '--apiRootPath',
+    argv.apiRootPath ?? '/',
+  ]
 
   if (argv.port) {
     args.push('--apiPort', String(argv.port))

packages/cli/src/commands/serveBothHandler.ts

@@ -12,7 +12,12 @@ import {
 } from '@cedarjs/api-server/cliHelpers'
 import { formatRunBinCommand } from '@cedarjs/cli-helpers/packageManager/display'
 import { runBin } from '@cedarjs/cli-helpers/packageManager/exec'
-import { getConfig, getPaths } from '@cedarjs/project-config'
+import {
+  getConfig,
+  getOTelImportArgs,
+  getPaths,
+  quoteOTelImportArgs,
+} from '@cedarjs/project-config'
 import { errorTelemetry } from '@cedarjs/telemetry'
 
 import { exitWithError } from '../lib/exit.js'
@@ -72,7 +77,13 @@ export const bothServerFileHandler = async (argv: ServeBothArgv) => {
       [
         {
           name: 'api',
-          command: `${formatRunBinCommand('node', [path.join('dist', 'server.js'), '--apiPort', String(argv.apiPort), '--apiHost', argv.apiHost, '--apiRootPath', apiRootPath])}`,
+          // The `--import` argument is a `file://` URL, which can contain
+          // characters that are special to the shell `concurrently` runs
+          // this command through. `quoteOTelImportArgs` picks the quoting
+          // that suits the platform's shell: single quotes (with embedded
+          // quotes escaped) for POSIX `sh`, double quotes for Windows
+          // `cmd.exe`.
+          command: `${formatRunBinCommand('node', [...quoteOTelImportArgs(getOTelImportArgs()), path.join('dist', 'server.js'), '--apiPort', String(argv.apiPort), '--apiHost', argv.apiHost, '--apiRootPath', apiRootPath])}`,
           cwd: getPaths().api.base,
           prefixColor: 'cyan',
         },

packages/cli/src/telemetry/index.ts

@@ -28,6 +28,13 @@ export async function startTelemetry() {
   }
   isStarted = true
 
+  // Mark this process as having the CLI's tracer provider registered
+  // globally, so in-process server launches know to skip importing the
+  // project's own OpenTelemetry setup file (it could not become the global
+  // provider here, and its spans would be misrouted into CLI telemetry).
+  // See `isCLITelemetryActive()` in `@cedarjs/project-config`.
+  globalThis.__CEDAR_CLI_TELEMETRY__ = true
+
   try {
     diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR)
 

packages/jobs/src/bins/cedar-jobs.ts

@@ -13,6 +13,8 @@ import { setTimeout } from 'node:timers'
 import { hideBin } from 'yargs/helpers'
 import yargs from 'yargs/yargs'
 
+import { getOTelImportArgs } from '@cedarjs/project-config'
+
 import { DEFAULT_LOGGER, PROCESS_TITLE_PREFIX } from '../consts.js'
 import { loadJobsManager } from '../loaders.js'
 import { setupEnv } from '../setupEnv.js'
@@ -121,6 +123,11 @@ export const startWorkers = ({
 }) => {
   logger.warn(`Starting ${numWorkers.length} worker(s)...`)
 
+  // Computed once here instead of inside the `map` below, so `cedar.toml`
+  // isn't parsed and the setup file checked once per worker, and a missing
+  // setup file doesn't warn once per worker.
+  const otelImportArgs = getOTelImportArgs()
+
   return numWorkers.map(([index, id]) => {
     // list of args to send to the forked worker script
     const workerArgs = ['--index', index.toString(), '--id', id.toString()]
@@ -134,6 +141,9 @@ export const startWorkers = ({
       detached: detach,
       stdio: detach ? 'ignore' : 'inherit',
       env: process.env,
+      // The OpenTelemetry SDK setup must be loaded before the app's own
+      // modules are imported, or the instrumentations can't patch them
+      execArgv: process.execArgv.concat(otelImportArgs),
     })
 
     if (detach) {