add gracefulWorkerRestartTimeout node renderer config

Author: AbanoubGhadban

react_on_rails_pro/docs/installation.md

@@ -129,6 +129,9 @@ const config = {
   // allWorkersRestartInterval: 15,
   // time in minutes between each worker restarting when restarting all workers
   // delayBetweenIndividualWorkerRestarts: 2,
+  // Also, you can set he parameter gracefulWorkerRestartTimeout to force the worker to restart
+  // If it's the time for the worker to restart, the worker waits until it serves all active requests before restarting
+  // If a worker stuck because of a memory leakage or an infinite loop, you can set a timeout that master waits for it before killing the worker
 }
 
 // Renderer detects a total number of CPUs on virtual hostings like Heroku

react_on_rails_pro/docs/node-renderer/js-configuration.md

@@ -22,6 +22,7 @@ Here are the options available for the JavaScript renderer configuration object,
 If no password is set, no authentication will be required.
 1. **allWorkersRestartInterval** (default: `env.RENDERER_ALL_WORKERS_RESTART_INTERVAL`) - Interval in minutes between scheduled restarts of all workers. By default restarts are not enabled. If restarts are enabled, `delayBetweenIndividualWorkerRestarts` should also be set.
 1. **delayBetweenIndividualWorkerRestarts** (default: `env.RENDERER_DELAY_BETWEEN_INDIVIDUAL_WORKER_RESTARTS`) - Interval in minutes between individual worker restarts (when cluster restart is triggered). By default restarts are not enabled. If restarts are enabled, `allWorkersRestartInterval` should also be set.
+1. **gracefulWorkerRestartTimeout**: (default: `env.GRACEFUL_WORKER_RESTART_TIMEOUT`) - Time in seconds that master waits for worker to gracefully restart (after serving all active requests) before killing it. You need to use this config to avoid situations where worker stucks at an inifite loop and doesn't restart. This config is only usable if worker restart is enabled. This time is counted starting from the time when should the worker restart, if that timeout value passed without restarting the worker, the worker is killed.
 1. **maxDebugSnippetLength** (default: 1000) - If the rendering request is longer than this, it will be truncated in exception and logging messages.
 1. **supportModules** - (default: `env.RENDERER_SUPPORT_MODULES || null`) - If set to true, `supportModules` enables the server-bundle code to call a default set of NodeJS global objects and functions that get added to the VM context: 
 `{ Buffer, TextDecoder, TextEncoder, URLSearchParams, ReadableStream, process, setTimeout, setInterval, setImmediate, clearTimeout, clearInterval, clearImmediate, queueMicrotask }`.

react_on_rails_pro/docs/node-renderer/troubleshooting.md

@@ -1,3 +1,5 @@
 # Node Renderer Troubleshooting
 
 - If you enabled restarts (having `allWorkersRestartInterval` and `delayBetweenIndividualWorkerRestarts`), you should set it with a high number to avoid the app from crashing because all Node renderer workers are stopped/killed.
+
+- If your app contains streamed pages that take too much time to be streamed to the client, ensure to not set the `gracefulWorkerRestartTimeout` parameter or set to a high number, so the worker is not killed while it's still serving an active request.

react_on_rails_pro/packages/node-renderer/src/master.ts

@@ -19,7 +19,12 @@ export = function masterRun(runningConfig?: Partial<Config>) {
 
   // Store config in app state. From now it can be loaded by any module using getConfig():
   const config = buildConfig(runningConfig);
-  const { workersCount, allWorkersRestartInterval, delayBetweenIndividualWorkerRestarts } = config;
+  const {
+    workersCount,
+    allWorkersRestartInterval,
+    delayBetweenIndividualWorkerRestarts,
+    gracefulWorkerRestartTimeout,
+  } = config;
 
   logSanitizedConfig();
 
@@ -51,7 +56,7 @@ export = function masterRun(runningConfig?: Partial<Config>) {
 
     const allWorkersRestartIntervalMS = allWorkersRestartInterval * MILLISECONDS_IN_MINUTE;
     const scheduleWorkersRestart = () => {
-      void restartWorkers(delayBetweenIndividualWorkerRestarts).finally(() => {
+      void restartWorkers(delayBetweenIndividualWorkerRestarts, gracefulWorkerRestartTimeout).finally(() => {
         setTimeout(scheduleWorkersRestart, allWorkersRestartIntervalMS);
       });
     };

react_on_rails_pro/packages/node-renderer/src/master/restartWorkers.ts

@@ -15,7 +15,10 @@ declare module 'cluster' {
   }
 }
 
-export = async function restartWorkers(delayBetweenIndividualWorkerRestarts: number) {
+export = async function restartWorkers(
+  delayBetweenIndividualWorkerRestarts: number,
+  gracefulWorkerRestartTimeout: number | undefined,
+) {
   log.info('Started scheduled restart of workers');
 
   if (!cluster.workers) {
@@ -39,12 +42,15 @@ export = async function restartWorkers(delayBetweenIndividualWorkerRestarts: num
       };
       worker.on('exit', onExit);
 
-      timeout = setTimeout(() => {
-        log.debug('Worker #%d timed out, forcing kill it', worker.id);
-        worker.destroy();
-        worker.off('exit', onExit);
-        resolve();
-      }, 100_000);
+      // Zero means no timeout
+      if (gracefulWorkerRestartTimeout) {
+        timeout = setTimeout(() => {
+          log.debug('Worker #%d timed out, forcing kill it', worker.id);
+          worker.destroy();
+          worker.off('exit', onExit);
+          resolve();
+        }, gracefulWorkerRestartTimeout);
+      }
     });
     // eslint-disable-next-line no-await-in-loop
     await new Promise((resolve) => {

react_on_rails_pro/packages/node-renderer/src/shared/configBuilder.ts

@@ -58,6 +58,9 @@ export interface Config {
   allWorkersRestartInterval: number | undefined;
   // Time in minutes between each worker restarting when restarting all workers
   delayBetweenIndividualWorkerRestarts: number | undefined;
+  // Time in seconds to wait for worker to restart before killing it
+  // Set it to 0 or undefined to never kill the worker
+  gracefulWorkerRestartTimeout: number | undefined;
   // If the rendering request is longer than this, it will be truncated in exception and logging messages
   maxDebugSnippetLength: number;
   // @deprecated See https://www.shakacode.com/react-on-rails-pro/docs/node-renderer/error-reporting-and-tracing.
@@ -165,6 +168,10 @@ const defaultConfig: Config = {
     ? parseInt(env.RENDERER_DELAY_BETWEEN_INDIVIDUAL_WORKER_RESTARTS, 10)
     : undefined,
 
+  gracefulWorkerRestartTimeout: env.GRACEFUL_WORKER_RESTART_TIMEOUT
+    ? parseInt(env.GRACEFUL_WORKER_RESTART_TIMEOUT, 10)
+    : undefined,
+
   maxDebugSnippetLength: MAX_DEBUG_SNIPPET_LENGTH,
 
   // default to true if empty, otherwise it is set to false
@@ -195,6 +202,8 @@ function envValuesUsed() {
     RENDERER_DELAY_BETWEEN_INDIVIDUAL_WORKER_RESTARTS:
       !userConfig.delayBetweenIndividualWorkerRestarts &&
       env.RENDERER_DELAY_BETWEEN_INDIVIDUAL_WORKER_RESTARTS,
+    GRACEFUL_WORKER_RESTART_TIMEOUT:
+      !userConfig.gracefulWorkerRestartTimeout && env.GRACEFUL_WORKER_RESTART_TIMEOUT,
     INCLUDE_TIMER_POLYFILLS: !('includeTimerPolyfills' in userConfig) && env.INCLUDE_TIMER_POLYFILLS,
     REPLAY_SERVER_ASYNC_OPERATION_LOGS:
       !userConfig.replayServerAsyncOperationLogs && env.REPLAY_SERVER_ASYNC_OPERATION_LOGS,
@@ -209,6 +218,7 @@ function sanitizedSettings(aConfig: Partial<Config> | undefined, defaultValue?:
         password: aConfig.password != null ? '<MASKED>' : defaultValue,
         allWorkersRestartInterval: aConfig.allWorkersRestartInterval || defaultValue,
         delayBetweenIndividualWorkerRestarts: aConfig.delayBetweenIndividualWorkerRestarts || defaultValue,
+        gracefulWorkerRestartTimeout: aConfig.gracefulWorkerRestartTimeout || defaultValue,
       }
     : {};
 }