Prevent fluid-build workers from exhausting all of memory (#21202)

## Description

This makes three changes to how fluid-build handles memory use:

1. When setting an infinite limit (unlimited) memory, encode that as
infinity bytes, instead of -1 bytes to simply logic and make semantics
more clear.
2. Use os.freemem to lower worker memory use limit as a heuristic
3. If there is less than 2 GB free, don't reuse the worker.

Also more documentation is added, including help text for the previously
undocumented --worker flag.

These changes are sufficient to make the default configuration of
fluid-build with --worker not run my system out of memory.

My system has 64 GB or ram, and was running out of memory when using
--worker to clean build client before this change.
With this change my system uses almost all the memory, and just swaps a
tiny bit and runs about as fast as with a manually tuned
-workerMemoryLimitMB 2000 (162 seconds).
This is faster than without worker (~200 seconds) which uses far less
memory.

Generally systems with more logical cpu cores (I have 32) get more
memory consumption with fluid-build due to having more threads and thus
more workers. This change should prevent such high core count systems
from running out of memory in most cases when using --worker if they
have enough memory to do a normal parallel build.

build-tools/packages/build-tools/src/fluidBuild/options.ts

@@ -58,7 +58,7 @@ export const options: FastBuildOptions = {
 	all: false,
 	worker: false,
 	workerThreads: false,
-	workerMemoryLimit: -1,
+	workerMemoryLimit: Number.POSITIVE_INFINITY,
 };
 
 // This string is duplicated in the readme: update readme if changing this.
@@ -85,6 +85,7 @@ Options:
      --symlink:full   Fix symlink between packages across monorepo (full mode). This symlinks everything in the repo together. CI does not ensure this configuration is functional, so it may or may not work.
      --uninstall      Clean all node_modules. This errors if some node-nodules folders do not exists: if hitting this limitation you can do an install first to work around it.
      --vscode         Output error message to work with default problem matcher in vscode
+     --worker         Reuse worker threads for some tasks, increasing memory use but lowering overhead.
 ${commonOptionString}
 `,
 	);

build-tools/packages/build-tools/src/fluidBuild/tasks/workers/workerPool.ts

@@ -3,10 +3,11 @@
  * Licensed under the MIT License.
  */
 
-import { ChildProcess, fork } from "child_process";
-import { EventEmitter } from "events";
-import { Readable } from "stream";
-import { Worker } from "worker_threads";
+import { ChildProcess, fork } from "node:child_process";
+import { EventEmitter } from "node:events";
+import { freemem } from "node:os";
+import { Readable } from "node:stream";
+import { Worker } from "node:worker_threads";
 
 import { WorkerExecResult, WorkerMessage } from "./worker";
 
@@ -36,7 +37,7 @@ export class WorkerPool {
 		if (!worker) {
 			worker = fork(
 				`${__dirname}/worker.js`,
-				this.memoryUsageLimit !== -1 ? ["--memoryUsage"] : undefined,
+				this.memoryUsageLimit !== Number.POSITIVE_INFINITY ? ["--memoryUsage"] : undefined,
 				{ silent: true },
 			);
 		}
@@ -107,11 +108,21 @@ export class WorkerPool {
 					worker.send(workerMessage);
 				});
 
+				// Workers accumulate memory use over time.
+				// Since recreating workers fixes this, but takes time,
+				// recreate them only when the memory use becomes too high.
+
+				// As a heuristic to avoid memory pressure, lower threshold if running out of memory.
+				const currentMemoryLimit = Math.min(this.memoryUsageLimit, freemem());
+
 				if (
-					this.memoryUsageLimit >= 0 &&
-					(res.memoryUsage?.rss ?? 0) > this.memoryUsageLimit
+					// Don't keep worker if using more than currentMemoryLimit bytes of memory.
+					(res.memoryUsage?.rss ?? 0) > currentMemoryLimit ||
+					// In case memoryUsage is not available,
+					// or as a last result when something other than this worker is using up all the memory
+					// kill the worker if there is less than 2 GB of memory free.
+					freemem() < 2 * 1024 * 1024 * 1024
 				) {
-					// Don't keep worker using more then 1GB of memory
 					worker.kill();
 				} else {
 					this.processWorkerPool.push(worker);