add gerrit config compilation

msukkari · msukkari · commit 7a8ac0a9c63f · 2025-02-09T19:47:08.000-08:00
diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts
@@ -5,7 +5,7 @@ import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
 import { createLogger } from "./logger.js";
 import os from 'os';
 import { Redis } from 'ioredis';
-import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig } from "./repoCompileUtils.js";
+import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig } from "./repoCompileUtils.js";
 
 interface IConnectionManager {
     scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -89,11 +89,14 @@ export class ConnectionManager implements IConnectionManager {
                 case 'gitea': {
                     return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
                 }
+                case 'gerrit': {
+                    return await compileGerritConfig(config, job.data.connectionId, orgId);
+                }
                 default: {
                     return [];
                 }
             }
-        )();
+        })();
 
         // Filter out any duplicates by external_id and external_codeHostUrl.
         repoData.filter((repo, index, self) => {
diff --git a/packages/backend/src/gerrit.ts b/packages/backend/src/gerrit.ts
@@ -1,8 +1,7 @@
 import fetch from 'cross-fetch';
 import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
-import { AppContext, GitRepository } from './types.js';
 import { createLogger } from './logger.js';
-import path from 'path';
+import micromatch from "micromatch";
 import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
 
 // https://gerrit-review.googlesource.com/Documentation/rest-api.html
@@ -16,19 +15,26 @@ interface GerritProjectInfo {
    web_links?: GerritWebLink[];
 }
 
+interface GerritProject {
+   name: string;
+   id: string;
+   state?: string;
+   web_links?: GerritWebLink[];
+}
+
 interface GerritWebLink {
    name: string;
    url: string;
 }
 
 const logger = createLogger('Gerrit');
 
-export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext): Promise<GitRepository[]> => {
+export const getGerritReposFromConfig = async (config: GerritConfig): Promise<GerritProject[]> => {
 
    const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
    const hostname = new URL(config.url).hostname;
 
-   const { durationMs, data: projects } = await measure(async () => {
+   let { durationMs, data: projects } = await measure(async () => {
       try {
          return fetchAllProjects(url)
       } catch (err) {
@@ -42,67 +48,29 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
    }
 
    // exclude "All-Projects" and "All-Users" projects
-   delete projects['All-Projects'];
-   delete projects['All-Users'];
-   delete projects['All-Avatars']
-   delete projects['All-Archived-Projects']
-
-   logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
-
-   let repos: GitRepository[] = Object.keys(projects).map((projectName) => {
-      const project = projects[projectName];
-      let webUrl = "https://www.gerritcodereview.com/";
-      // Gerrit projects can have multiple web links; use the first one
-      if (project.web_links) {
-         const webLink = project.web_links[0];
-         if (webLink) {
-            webUrl = webLink.url;
-         }
-      }
-      const repoId = `${hostname}/${projectName}`;
-      const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
-
-      const cloneUrl = `${url}${encodeURIComponent(projectName)}`;
-
-      return {
-         vcs: 'git',
-         codeHost: 'gerrit',
-         name: projectName,
-         id: repoId,
-         cloneUrl: cloneUrl,
-         path: repoPath,
-         isStale: false, // Gerrit projects are typically not stale
-         isFork: false, // Gerrit doesn't have forks in the same way as GitHub
-         isArchived: false,
-         gitConfigMetadata: {
-            // Gerrit uses Gitiles for web UI. This can sometimes be "browse" type in zoekt
-            'zoekt.web-url-type': 'gitiles',
-            'zoekt.web-url': webUrl,
-            'zoekt.name': repoId,
-            'zoekt.archived': marshalBool(false),
-            'zoekt.fork': marshalBool(false),
-            'zoekt.public': marshalBool(true), // Assuming projects are public; adjust as needed
-         },
-         branches: [],
-         tags: []
-      } satisfies GitRepository;
-   });
-
+   const excludedProjects = ['All-Projects', 'All-Users', 'All-Avatars', 'All-Archived-Projects'];
+   projects = projects.filter(project => !excludedProjects.includes(project.name));
+   
    // include repos by glob if specified in config
    if (config.projects) {
-      repos = includeReposByName(repos, config.projects);
+      projects = projects.filter((project) => {
+         return micromatch.isMatch(project.name, config.projects!);
+      });
    }
-
+   
    if (config.exclude && config.exclude.projects) {
-      repos = excludeReposByName(repos, config.exclude.projects);
+      projects = projects.filter((project) => {
+         return !micromatch.isMatch(project.name, config.exclude!.projects!);
+      });
    }
 
-   return repos;
+   logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
+   return projects;
 };
 
-const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
+const fetchAllProjects = async (url: string): Promise<GerritProject[]> => {
    const projectsEndpoint = `${url}projects/`;
-   let allProjects: GerritProjects = {};
+   let allProjects: GerritProject[] = [];
    let start = 0; // Start offset for pagination
    let hasMoreProjects = true;
 
@@ -119,8 +87,15 @@ const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
       const jsonText = text.replace(")]}'\n", ''); // Remove XSSI protection prefix
       const data: GerritProjects = JSON.parse(jsonText);
 
-      // Merge the current batch of projects with allProjects
-      Object.assign(allProjects, data);
+      // Add fetched projects to allProjects
+      for (const [projectName, projectInfo] of Object.entries(data)) {
+         allProjects.push({
+            name: projectName,
+            id: projectInfo.id,
+            state: projectInfo.state,
+            web_links: projectInfo.web_links
+         })
+      }
 
       // Check if there are more projects to fetch
       hasMoreProjects = Object.values(data).some(
diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts
@@ -2,10 +2,11 @@ import { GithubConnectionConfig } from '@sourcebot/schemas/v3/github.type';
 import { getGitHubReposFromConfig } from "./github.js";
 import { getGitLabReposFromConfig } from "./gitlab.js";
 import { getGiteaReposFromConfig } from "./gitea.js";
+import { getGerritReposFromConfig } from "./gerrit.js";
 import { Prisma, PrismaClient } from '@sourcebot/db';
 import { WithRequired } from "./types.js"
 import { marshalBool } from "./utils.js";
-import { GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
+import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
 
 export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
 
@@ -148,6 +149,60 @@ export const compileGiteaConfig = async (
             },
         };
 
+        return record;
+    })
+}
+
+export const compileGerritConfig = async (
+    config: GerritConnectionConfig,
+    connectionId: number,
+    orgId: number) => {
+
+    const gerritRepos = await getGerritReposFromConfig(config);
+    const hostUrl = config.url ?? 'https://gerritcodereview.com';
+    const hostname = config.url ? new URL(config.url).hostname : 'gerritcodereview.com';
+
+    return gerritRepos.map((project) => {
+        const repoId = `${hostname}/${project.name}`;
+        const cloneUrl = new URL(`${config.url}/${encodeURIComponent(project.name)}`);
+
+        let webUrl = "https://www.gerritcodereview.com/";
+        // Gerrit projects can have multiple web links; use the first one
+        if (project.web_links) {
+            const webLink = project.web_links[0];
+            if (webLink) {
+                webUrl = webLink.url;
+            }
+        }
+
+        const record: RepoData = {
+            external_id: project.id.toString(),
+            external_codeHostType: 'gerrit',
+            external_codeHostUrl: hostUrl,
+            cloneUrl: cloneUrl.toString(),
+            name: project.name,
+            isFork: false,
+            isArchived: false,
+            org: {
+                connect: {
+                    id: orgId,
+                },
+            },
+            connections: {
+                create: {
+                    connectionId: connectionId,
+                }
+            },
+            metadata: {
+                'zoekt.web-url-type': 'gitiles',
+                'zoekt.web-url': webUrl,
+                'zoekt.name': repoId,
+                'zoekt.archived': marshalBool(false),
+                'zoekt.fork': marshalBool(false),
+                'zoekt.public': marshalBool(true),
+            },
+        };
+
         return record;
     })
 }
diff --git a/packages/web/src/app/connections/quickActions.ts b/packages/web/src/app/connections/quickActions.ts
@@ -135,9 +135,15 @@ export const gerritQuickActions: QuickAction<GerritConnectionConfig>[] = [
     {
         fn: (previous: GerritConnectionConfig) => ({
             ...previous,
-            url: previous.url ?? "",
+            exclude: {
+                ...previous.exclude,
+                projects: [
+                    ...(previous.exclude?.projects ?? []),
+                    ""
+                ]
+            }
         }),
-        name: "Set a custom url",
+        name: "Exclude a project",
     }
 ]
 

Original file line number	Diff line number	Diff line change
`@@ -135,9 +135,15 @@ export const gerritQuickActions: QuickAction<GerritConnectionConfig>[] = [`
`135`	`135`	`{`
`136`	`136`	`fn: (previous: GerritConnectionConfig) => ({`
`137`	`137`	`...previous,`
`138`		`- url: previous.url ?? "",`
	`138`	`+ exclude: {`
	`139`	`+ ...previous.exclude,`
	`140`	`+ projects: [`
	`141`	`+ ...(previous.exclude?.projects ?? []),`
	`142`	`+ ""`
	`143`	`+ ]`
	`144`	`+ }`
`139`	`145`	`}),`
`140`		`- name: "Set a custom url",`
	`146`	`+ name: "Exclude a project",`
`141`	`147`	`}`
`142`	`148`	`]`
`143`	`149`