renames + add abortSignal

brendan-kellam · brendan-kellam · commit 5fe554e7da1d · 2025-10-15T11:23:34.000-07:00
diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts
@@ -10,7 +10,7 @@ type onProgressFn = (event: SimpleGitProgressEvent) => void;
  * Creates a simple-git client that has it's working directory
  * set to the given path.
  */
-const createGitClientForPath = (path: string, onProgress?: onProgressFn) => {
+const createGitClientForPath = (path: string, onProgress?: onProgressFn, signal?: AbortSignal) => {
     if (!existsSync(path)) {
         throw new Error(`Path ${path} does not exist`);
     }
@@ -19,6 +19,7 @@ const createGitClientForPath = (path: string, onProgress?: onProgressFn) => {
 
     const git = simpleGit({
         progress: onProgress,
+        abort: signal,
     })
         .env({
             ...process.env,
@@ -48,17 +49,19 @@ export const cloneRepository = async (
         authHeader,
         path,
         onProgress,
+        signal,
     }: {
         cloneUrl: string,
         authHeader?: string,
         path: string,
         onProgress?: onProgressFn
+        signal?: AbortSignal
     }
 ) => {
     try {
         await mkdir(path, { recursive: true });
 
-        const git = createGitClientForPath(path, onProgress);
+        const git = createGitClientForPath(path, onProgress, signal);
 
         const cloneArgs = [
             "--bare",
@@ -67,7 +70,11 @@ export const cloneRepository = async (
 
         await git.clone(cloneUrl, path, cloneArgs);
 
-        await unsetGitConfig(path, ["remote.origin.url"]);
+        await unsetGitConfig({
+            path,
+            keys: ["remote.origin.url"],
+            signal,
+        });
     } catch (error: unknown) {
         const baseLog = `Failed to clone repository: ${path}`;
 
@@ -88,15 +95,17 @@ export const fetchRepository = async (
         authHeader,
         path,
         onProgress,
+        signal,
     }: {
         cloneUrl: string,
         authHeader?: string,
         path: string,
-        onProgress?: onProgressFn
+        onProgress?: onProgressFn,
+        signal?: AbortSignal
     }
 ) => {
     try {
-        const git = createGitClientForPath(path, onProgress);
+        const git = createGitClientForPath(path, onProgress, signal);
 
         if (authHeader) {
             await git.addConfig("http.extraHeader", authHeader);
@@ -137,8 +146,19 @@ export const fetchRepository = async (
  * that do not exist yet. It will _not_ remove any existing keys that are not
  * present in gitConfig.
  */
-export const upsertGitConfig = async (path: string, gitConfig: Record<string, string>, onProgress?: onProgressFn) => {
-    const git = createGitClientForPath(path, onProgress);
+export const upsertGitConfig = async (
+    {
+        path,
+        gitConfig,
+        onProgress,
+        signal,
+    }: {
+        path: string,
+        gitConfig: Record<string, string>,
+        onProgress?: onProgressFn,
+        signal?: AbortSignal
+    }) => {
+    const git = createGitClientForPath(path, onProgress, signal);
 
     try {
         for (const [key, value] of Object.entries(gitConfig)) {
@@ -157,8 +177,19 @@ export const upsertGitConfig = async (path: string, gitConfig: Record<string, st
  * Unsets the specified keys in the git config for the repo at the given path.
  * If a key is not set, this is a no-op.
  */
-export const unsetGitConfig = async (path: string, keys: string[], onProgress?: onProgressFn) => {
-    const git = createGitClientForPath(path, onProgress);
+export const unsetGitConfig = async (
+    {
+        path,
+        keys,
+        onProgress,
+        signal,
+    }: {
+        path: string,
+        keys: string[],
+        onProgress?: onProgressFn,
+        signal?: AbortSignal
+    }) => {
+    const git = createGitClientForPath(path, onProgress, signal);
 
     try {
         const configList = await git.listConfig();
@@ -181,8 +212,16 @@ export const unsetGitConfig = async (path: string, keys: string[], onProgress?:
 /**
  * Returns true if `path` is the _root_ of a git repository.
  */
-export const isPathAValidGitRepoRoot = async (path: string, onProgress?: onProgressFn) => {
-    const git = createGitClientForPath(path, onProgress);
+export const isPathAValidGitRepoRoot = async ({
+    path,
+    onProgress,
+    signal,
+}: {
+    path: string,
+    onProgress?: onProgressFn,
+    signal?: AbortSignal
+}) => {
+    const git = createGitClientForPath(path, onProgress, signal);
 
     try {
         return git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT);
diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts
@@ -11,9 +11,8 @@ import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.
 import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
 import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
 import { env } from "./env.js";
-import { IndexSyncer } from "./indexSyncer.js";
+import { RepoIndexManager } from "./repoIndexManager.js";
 import { PromClient } from './promClient.js';
-import { RepoManager } from './repoManager.js';
 
 
 const logger = createLogger('backend-entrypoint');
@@ -60,16 +59,12 @@ const promClient = new PromClient();
 const settings = await getSettings(env.CONFIG_PATH);
 
 const connectionManager = new ConnectionManager(prisma, settings, redis);
-const repoManager = new RepoManager(prisma, settings, redis, promClient);
 const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis);
 const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis);
-const indexSyncer = new IndexSyncer(prisma, settings, redis);
-
-// await repoManager.validateIndexedReposHaveShards();
+const repoIndexManager = new RepoIndexManager(prisma, settings, redis);
 
 connectionManager.startScheduler();
-// repoManager.startScheduler();
-indexSyncer.startScheduler();
+repoIndexManager.startScheduler();
 
 if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && !hasEntitlement('permission-syncing')) {
     logger.error('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.');
@@ -88,8 +83,7 @@ const cleanup = async (signal: string) => {
     try {
         await Promise.race([
             Promise.all([
-                indexSyncer.dispose(),
-                repoManager.dispose(),
+                repoIndexManager.dispose(),
                 connectionManager.dispose(),
                 repoPermissionSyncer.dispose(),
                 userPermissionSyncer.dispose(),
diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts
@@ -497,7 +497,9 @@ export const compileGenericGitHostConfig_file = async (
     };
     
     await Promise.all(repoPaths.map(async (repoPath) => {
-        const isGitRepo = await isPathAValidGitRepoRoot(repoPath);
+        const isGitRepo = await isPathAValidGitRepoRoot({
+            path: repoPath,
+        });
         if (!isGitRepo) {
             logger.warn(`Skipping ${repoPath} - not a git repository.`);
             notFound.repos.push(repoPath);
diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts
@@ -12,7 +12,7 @@ import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
 import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
 import { indexGitRepository } from './zoekt.js';
 
-const LOG_TAG = 'index-syncer';
+const LOG_TAG = 'repo-index-manager';
 const logger = createLogger(LOG_TAG);
 const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`);
 
@@ -25,7 +25,18 @@ type JobPayload = {
 
 const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 6; // 6 hour indexing timeout
 
-export class IndexSyncer {
+/**
+ * Manages the lifecycle of repository data on disk, including git working copies
+ * and search index shards. Handles both indexing operations (cloning/fetching repos
+ * and building search indexes) and cleanup operations (removing orphaned repos and
+ * their associated data).
+ * 
+ * Uses a job queue system to process indexing and cleanup tasks asynchronously,
+ * with configurable concurrency limits and retry logic. Automatically schedules
+ * re-indexing of repos based on configured intervals and manages garbage collection
+ * of repos that are no longer connected to any source.
+ */
+export class RepoIndexManager {
     private interval?: NodeJS.Timeout;
     private queue: Queue<JobPayload>;
     private worker: Worker<JobPayload>;
@@ -37,7 +48,7 @@ export class IndexSyncer {
     ) {
         this.queue = new Queue<JobPayload>({
             redis,
-            namespace: 'index-sync-queue',
+            namespace: 'repo-index-queue',
             jobTimeoutMs: JOB_TIMEOUT_MS,
             maxAttempts: 3,
             logger: env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true',
@@ -210,6 +221,7 @@ export class IndexSyncer {
         const logger = createJobLogger(id);
         logger.info(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`);
 
+
         const { repo, type: jobType } = await this.db.repoJob.update({
             where: {
                 id,
@@ -231,14 +243,28 @@ export class IndexSyncer {
             }
         });
 
-        if (jobType === RepoJobType.INDEX) {
-            await this.indexRepository(repo, logger);
-        } else if (jobType === RepoJobType.CLEANUP) {
-            await this.cleanupRepository(repo, logger);
+        const abortController = new AbortController();
+        const signalHandler = () => {
+            logger.info(`Received shutdown signal, aborting...`);
+            abortController.abort(); // This cancels all operations
+        };
+
+        process.on('SIGTERM', signalHandler);
+        process.on('SIGINT', signalHandler);
+
+        try {
+            if (jobType === RepoJobType.INDEX) {
+                await this.indexRepository(repo, logger, abortController.signal);
+            } else if (jobType === RepoJobType.CLEANUP) {
+                await this.cleanupRepository(repo, logger);
+            }
+        } finally {
+            process.off('SIGTERM', signalHandler);
+            process.off('SIGINT', signalHandler);
         }
     }
 
-    private async indexRepository(repo: RepoWithConnections, logger: Logger) {
+    private async indexRepository(repo: RepoWithConnections, logger: Logger, signal: AbortSignal) {
         const { path: repoPath, isReadOnly } = getRepoPath(repo);
 
         const metadata = repoMetadataSchema.parse(repo.metadata);
@@ -250,9 +276,16 @@ export class IndexSyncer {
         // If the repo path exists but it is not a valid git repository root, this indicates
         // that the repository is in a bad state. To fix, we remove the directory and perform
         // a fresh clone.
-        if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot(repoPath)) && !isReadOnly) {
-            logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`);
-            await rm(repoPath, { recursive: true, force: true });
+        if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot( { path: repoPath } ))) {
+            const isValidGitRepo = await isPathAValidGitRepoRoot({
+                path: repoPath,
+                signal,
+            });
+
+            if (!isValidGitRepo && !isReadOnly) {
+                logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`);
+                await rm(repoPath, { recursive: true, force: true });
+            }
         }
 
         if (existsSync(repoPath) && !isReadOnly) {
@@ -262,7 +295,11 @@ export class IndexSyncer {
             // to unset this key since it is no longer needed, hence this line.
             // This will no-op if the key is already unset.
             // @see: https://github.com/sourcebot-dev/sourcebot/pull/483
-            await unsetGitConfig(repoPath, ["remote.origin.url"]);
+            await unsetGitConfig({
+                path: repoPath,
+                keys: ["remote.origin.url"],
+                signal,
+            });
 
             logger.info(`Fetching ${repo.name} (id: ${repo.id})...`);
             const { durationMs } = await measure(() => fetchRepository({
@@ -271,7 +308,8 @@ export class IndexSyncer {
                 path: repoPath,
                 onProgress: ({ method, stage, progress }) => {
                     logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`)
-                }
+                },
+                signal,
             }));
             const fetchDuration_s = durationMs / 1000;
 
@@ -287,7 +325,8 @@ export class IndexSyncer {
                 path: repoPath,
                 onProgress: ({ method, stage, progress }) => {
                     logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`)
-                }
+                },
+                signal
             }));
             const cloneDuration_s = durationMs / 1000;
 
@@ -299,11 +338,15 @@ export class IndexSyncer {
         // This ensures that the git config is always up to date for whatever we
         // have in the DB.
         if (metadata.gitConfig && !isReadOnly) {
-            await upsertGitConfig(repoPath, metadata.gitConfig);
+            await upsertGitConfig({
+                path: repoPath,
+                gitConfig: metadata.gitConfig,
+                signal,
+            });
         }
 
         logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
-        const { durationMs } = await measure(() => indexGitRepository(repo, this.settings));
+        const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
         const indexDuration_s = durationMs / 1000;
         logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);
     }
diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts
diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts
