Add concept of 'operations' to track long-running tasks (like cloning)

Author: brendan-kellam

packages/backend/package.json

@@ -12,6 +12,7 @@
     "devDependencies": {
         "@types/argparse": "^2.0.16",
         "@types/node": "^22.7.5",
+        "@types/uuid": "^10.0.0",
         "json-schema-to-typescript": "^15.0.2",
         "tsx": "^4.19.1",
         "typescript": "^5.6.2"
@@ -21,6 +22,7 @@
         "argparse": "^2.0.1",
         "lowdb": "^7.0.1",
         "simple-git": "^3.27.0",
+        "uuid": "^10.0.0",
         "winston": "^3.15.0"
     }
 }

packages/backend/src/db.ts

@@ -1,19 +1,39 @@
 import { JSONFilePreset } from "lowdb/node";
 import { type Low } from "lowdb";
-import { AppContext, Repository } from "./types.js";
+import { AppContext, Operation, Repository } from "./types.js";
+import { v4 as uuid } from 'uuid';
 
 type Schema = {
     repos: {
         [key: string]: Repository;
+    },
+    operations: {
+        [key: string]: Operation;
     }
 }
 
 export type Database = Low<Schema>;
 
 export const loadDB = async (ctx: AppContext): Promise<Database> => {
-    const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, { repos: {} });
+    const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, { repos: {}, operations: {} });
     return db;
 }
+
+export const updateOperation = async (id: string, data: Partial<Operation>, db: Database) => {
+    db.data.operations[id] = {
+        ...db.data.operations[id],
+        ...data,
+    }
+    await db.write();
+}
+
+export const createOperation = async (data: Operation, db: Database) => {
+    const id = uuid();
+    db.data.operations[id] = data;
+    await db.write();
+    return id;
+}
+
 export const updateRepository = async (repoId: string, data: Partial<Repository>, db: Database) => {
     db.data.repos[repoId] = {
         ...db.data.repos[repoId],

packages/backend/src/index.ts

@@ -8,7 +8,7 @@ import { getGitHubReposFromConfig } from "./github.js";
 import { AppContext, Repository } from "./types.js";
 import { cloneRepository, fetchRepository } from "./git.js";
 import { createLogger } from "./logger.js";
-import { Database, loadDB } from './db.js';
+import { createOperation, createRepository, Database, loadDB, updateOperation, updateRepository } from './db.js';
 import { measure } from "./utils.js";
 import { REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js";
 
@@ -196,19 +196,38 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
 
             logger.info(`Indexing ${repo.id}...`);
 
-            try {
-                if (existsSync(repo.path)) {
-                    await fetchRepository(repo);
-                } else {
-                    await cloneRepository(repo);
+            if (existsSync(repo.path)) {
+                await fetchRepository(repo);
+            } else {
+                const operationId = await createOperation({
+                    type: 'clone',
+                    progress: 0,
+                    status: 'running',
+                    repoId: repo.id,
+                    startedAtDate: new Date().toUTCString(),
+                }, db);
+
+                try {
+                    await cloneRepository(repo, (progress) => {
+                        updateOperation(operationId, {
+                            progress: progress,
+                        }, db);
+                    });
+                } catch (err: any) {
+                    console.error(err);
+                    await updateOperation(operationId, {
+                        status: 'failed',
+                        message: err.message,
+                    }, db);
+                    throw err;
                 }
 
-                await indexRepository(repo, context);
-            } catch (err: any) {
-                // @todo : better error handling here..
-                logger.error(err);
-                continue;
+                await updateOperation(operationId, {
+                    status: 'completed',
+                }, db);
             }
+
+            await indexRepository(repo, context);
 
             await db.update(({ repos }) => repos[repo.id].lastIndexedDate = new Date().toUTCString());
             logger.info(`Indexed ${repo.id}`);

packages/backend/src/types.ts

@@ -1,4 +1,13 @@
 
+export type Operation = {
+    type: 'clone',
+    status: 'running' | 'completed' | 'failed',
+    progress: number,
+    repoId: string;
+    startedAtDate: string;
+    message?: string;
+}
+
 export type Repository = {
     /**
      * Name of the repository (e.g., 'sourcebot-dev/sourcebot')

yarn.lock

@@ -1324,6 +1324,11 @@
   resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c"
   integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==
 
+"@types/uuid@^10.0.0":
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-10.0.0.tgz#e9c07fe50da0f53dc24970cca94d619ff03f6f6d"
+  integrity sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==
+
 "@typescript-eslint/eslint-plugin@^8.3.0":
   version "8.8.0"
   resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.8.0.tgz#b2b02a5447cdc885950eb256b3b8a97b92031bd3"
@@ -4599,6 +4604,11 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2:
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==
 
+uuid@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294"
+  integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==
+
 validate-npm-package-license@^3.0.1:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a"