[Feature]: Add ability to disable background processing and only run Web UI and API server #2901 (#2902)

* add server setting

* add logic to disable background processing

* update docs for new setting

* Rename to DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED

---------

Co-authored-by: Victor Skvortsov <vds003@gmail.com>

Author: james-boydell

docs/docs/reference/environment-variables.md

@@ -123,6 +123,7 @@ For more details on the options below, refer to the [server deployment](../guide
 - `DSTACK_DB_POOL_SIZE`{ #DSTACK_DB_POOL_SIZE } - The client DB connections pool size. Defaults to `20`,
 - `DSTACK_DB_MAX_OVERFLOW`{ #DSTACK_DB_MAX_OVERFLOW } - The client DB connections pool allowed overflow. Defaults to `20`.
 - `DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR } - The number of background jobs for processing server resources. Increase if you need to process more resources per server replica quickly. Defaults to `1`.
+- `DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED } - Disables background processing if set to any value. Useful to run only web frontend and API server.
 
 ??? info "Internal environment variables"
      The following environment variables are intended for development purposes:

src/dstack/_internal/server/app.py

@@ -151,7 +151,10 @@ async def lifespan(app: FastAPI):
     )
     if settings.SERVER_S3_BUCKET is not None or settings.SERVER_GCS_BUCKET is not None:
         init_default_storage()
-    scheduler = start_background_tasks()
+    if settings.SERVER_BACKGROUND_PROCESSING_ENABLED:
+        scheduler = start_background_tasks()
+    else:
+        logger.info("Background processing is disabled")
     dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)"
     logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False})
     logger.info(
@@ -161,7 +164,8 @@ async def lifespan(app: FastAPI):
     for func in _ON_STARTUP_HOOKS:
         await func(app)
     yield
-    scheduler.shutdown()
+    if settings.SERVER_BACKGROUND_PROCESSING_ENABLED:
+        scheduler.shutdown()
     await gateway_connections_pool.remove_all()
     service_conn_pool = await get_injector_from_app(app).get_service_connection_pool()
     await service_conn_pool.remove_all()

src/dstack/_internal/server/settings.py

@@ -42,6 +42,11 @@
     os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR", 1)
 )
 
+SERVER_BACKGROUND_PROCESSING_DISABLED = (
+    os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None
+)
+SERVER_BACKGROUND_PROCESSING_ENABLED = not SERVER_BACKGROUND_PROCESSING_DISABLED
+
 SERVER_EXECUTOR_MAX_WORKERS = int(os.getenv("DSTACK_SERVER_EXECUTOR_MAX_WORKERS", 128))
 
 MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25))
@@ -113,5 +118,5 @@
 
 UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None
 DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None
-SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE", None) is not None
-ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS", None) is not None
+SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE") is not None
+ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS") is not None