Add support for remote configs

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Added support for specifying urls for the `--configPath` option in the backend.
+
 ## [2.0.0] - 2024-10-17
 
 ### Added

packages/backend/src/index.ts

@@ -10,7 +10,7 @@ import { AppContext, Repository } from "./types.js";
 import { cloneRepository, fetchRepository } from "./git.js";
 import { createLogger } from "./logger.js";
 import { createRepository, Database, loadDB, updateRepository } from './db.js';
-import { measure } from "./utils.js";
+import { isRemotePath, measure } from "./utils.js";
 import { REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js";
 import stripJsonComments from 'strip-json-comments';
 
@@ -41,10 +41,22 @@ const indexRepository = async (repo: Repository, ctx: AppContext) => {
 }
 
 const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, ctx: AppContext) => {
-    const configContent = await readFile(configPath, { 
-        encoding: 'utf-8',
-        signal,
-    });
+    const configContent = await (async () => {
+        if (isRemotePath(configPath)) {
+            const response = await fetch(configPath, {
+                signal,
+            });
+            if (!response.ok) {
+                throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
+            }
+            return response.text();
+        } else {
+            return readFile(configPath, {
+                encoding: 'utf-8',
+                signal,
+            });
+        }
+    })();
 
     // @todo: we should validate the configuration file's structure here.
     const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfigurationSchema;
@@ -122,7 +134,7 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
     });
     const args = parser.parse_args() as Arguments;
 
-    if (!existsSync(args.configPath)) {
+    if (!isRemotePath(args.configPath) && !existsSync(args.configPath)) {
         console.error(`Config file ${args.configPath} does not exist`);
         process.exit(1);
     }
@@ -173,11 +185,13 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
             });
     }
 
-    // Re-sync on file changes
-    watch(args.configPath, () => {
-        logger.info(`Config file ${args.configPath} changed. Re-syncing...`);
-        _syncConfig();
-    });
+    // Re-sync on file changes if the config file is local
+    if (!isRemotePath(args.configPath)) {
+        watch(args.configPath, () => {
+            logger.info(`Config file ${args.configPath} changed. Re-syncing...`);
+            _syncConfig();
+        });
+    }
 
     // Re-sync every 24 hours
     setInterval(() => {

packages/backend/src/utils.ts

@@ -55,4 +55,8 @@ export const getTokenFromConfig = (token: string | { env: string }, ctx: AppCont
         throw new Error(`The environment variable '${token.env}' was referenced in ${ctx.configPath}, but was not set.`);
     }
     return tokenValue;
+}
+
+export const isRemotePath = (path: string) => {
+    return path.startsWith('https://') || path.startsWith('http://');
 }