Skip to content

Commit da33220

Browse files
msukkaribrendan-kellam
msukkari
and
brendan-kellam
authored
add back gitlab, gitea, and gerrit support (#184)
* add non github config definitions * refactor github config compilation to seperate file * add gitlab config compilation * Connection management (#183) * wip gitlab repo sync support * fix gitlab zoekt metadata * add gitea support * add gerrit support * Connection management (#183) * add gerrit config compilation * Connection management (#183) --------- Co-authored-by: Brendan Kellam <bshizzle1234@gmail.com>
1 parent e2e5433 commit da33220

30 files changed

+1576
-213
lines changed

‎packages/backend/src/connectionManager.ts

Lines changed: 21 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
55
import { createLogger } from "./logger.js";
66
import os from 'os';
77
import { Redis } from 'ioredis';
8-
import { marshalBool } from "./utils.js";
9-
import { getGitHubReposFromConfig } from "./github.js";
8+
import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig } from "./repoCompileUtils.js";
109

1110
interface IConnectionManager {
1211
scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -79,64 +78,28 @@ export class ConnectionManager implements IConnectionManager {
7978
// @note: We aren't actually doing anything with this atm.
8079
const abortController = new AbortController();
8180

82-
type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
83-
const repoData: RepoData[] = (
84-
await (async () => {
85-
switch (config.type) {
86-
case 'github': {
87-
const gitHubRepos = await getGitHubReposFromConfig(config, orgId, this.db, abortController.signal);
88-
const hostUrl = config.url ?? 'https://github.com';
89-
const hostname = config.url ? new URL(config.url).hostname : 'github.com';
90-
91-
return gitHubRepos.map((repo) => {
92-
const repoName = `${hostname}/${repo.full_name}`;
93-
const cloneUrl = new URL(repo.clone_url!);
94-
95-
const record: RepoData = {
96-
external_id: repo.id.toString(),
97-
external_codeHostType: 'github',
98-
external_codeHostUrl: hostUrl,
99-
cloneUrl: cloneUrl.toString(),
100-
imageUrl: repo.owner.avatar_url,
101-
name: repoName,
102-
isFork: repo.fork,
103-
isArchived: !!repo.archived,
104-
org: {
105-
connect: {
106-
id: orgId,
107-
},
108-
},
109-
connections: {
110-
create: {
111-
connectionId: job.data.connectionId,
112-
}
113-
},
114-
metadata: {
115-
'zoekt.web-url-type': 'github',
116-
'zoekt.web-url': repo.html_url,
117-
'zoekt.name': repoName,
118-
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
119-
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
120-
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
121-
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
122-
'zoekt.archived': marshalBool(repo.archived),
123-
'zoekt.fork': marshalBool(repo.fork),
124-
'zoekt.public': marshalBool(repo.private === false)
125-
},
126-
};
127-
128-
return record;
129-
})
130-
}
131-
case 'gitlab': {
132-
// @todo
133-
return [];
134-
}
81+
const repoData: RepoData[] = await (async () => {
82+
switch (config.type) {
83+
case 'github': {
84+
return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController);
85+
}
86+
case 'gitlab': {
87+
return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db);
88+
}
89+
case 'gitea': {
90+
return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
13591
}
136-
})()
137-
)
92+
case 'gerrit': {
93+
return await compileGerritConfig(config, job.data.connectionId, orgId);
94+
}
95+
default: {
96+
return [];
97+
}
98+
}
99+
})();
100+
138101
// Filter out any duplicates by external_id and external_codeHostUrl.
139-
.filter((repo, index, self) => {
102+
repoData.filter((repo, index, self) => {
140103
return index === self.findIndex(r =>
141104
r.external_id === repo.external_id &&
142105
r.external_codeHostUrl === repo.external_codeHostUrl

‎packages/backend/src/gerrit.ts

Lines changed: 33 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
import fetch from 'cross-fetch';
22
import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
3-
import { AppContext, GitRepository } from './types.js';
43
import { createLogger } from './logger.js';
5-
import path from 'path';
4+
import micromatch from "micromatch";
65
import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
76

87
// https://gerrit-review.googlesource.com/Documentation/rest-api.html
@@ -16,19 +15,26 @@ interface GerritProjectInfo {
1615
web_links?: GerritWebLink[];
1716
}
1817

18+
interface GerritProject {
19+
name: string;
20+
id: string;
21+
state?: string;
22+
web_links?: GerritWebLink[];
23+
}
24+
1925
interface GerritWebLink {
2026
name: string;
2127
url: string;
2228
}
2329

2430
const logger = createLogger('Gerrit');
2531

26-
export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext): Promise<GitRepository[]> => {
32+
export const getGerritReposFromConfig = async (config: GerritConfig): Promise<GerritProject[]> => {
2733

2834
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
2935
const hostname = new URL(config.url).hostname;
3036

31-
const { durationMs, data: projects } = await measure(async () => {
37+
let { durationMs, data: projects } = await measure(async () => {
3238
try {
3339
return fetchAllProjects(url)
3440
} catch (err) {
@@ -42,67 +48,29 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
4248
}
4349

4450
// exclude "All-Projects" and "All-Users" projects
45-
delete projects['All-Projects'];
46-
delete projects['All-Users'];
47-
delete projects['All-Avatars']
48-
delete projects['All-Archived-Projects']
49-
50-
logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
51-
52-
let repos: GitRepository[] = Object.keys(projects).map((projectName) => {
53-
const project = projects[projectName];
54-
let webUrl = "https://www.gerritcodereview.com/";
55-
// Gerrit projects can have multiple web links; use the first one
56-
if (project.web_links) {
57-
const webLink = project.web_links[0];
58-
if (webLink) {
59-
webUrl = webLink.url;
60-
}
61-
}
62-
const repoId = `${hostname}/${projectName}`;
63-
const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
64-
65-
const cloneUrl = `${url}${encodeURIComponent(projectName)}`;
66-
67-
return {
68-
vcs: 'git',
69-
codeHost: 'gerrit',
70-
name: projectName,
71-
id: repoId,
72-
cloneUrl: cloneUrl,
73-
path: repoPath,
74-
isStale: false, // Gerrit projects are typically not stale
75-
isFork: false, // Gerrit doesn't have forks in the same way as GitHub
76-
isArchived: false,
77-
gitConfigMetadata: {
78-
// Gerrit uses Gitiles for web UI. This can sometimes be "browse" type in zoekt
79-
'zoekt.web-url-type': 'gitiles',
80-
'zoekt.web-url': webUrl,
81-
'zoekt.name': repoId,
82-
'zoekt.archived': marshalBool(false),
83-
'zoekt.fork': marshalBool(false),
84-
'zoekt.public': marshalBool(true), // Assuming projects are public; adjust as needed
85-
},
86-
branches: [],
87-
tags: []
88-
} satisfies GitRepository;
89-
});
90-
51+
const excludedProjects = ['All-Projects', 'All-Users', 'All-Avatars', 'All-Archived-Projects'];
52+
projects = projects.filter(project => !excludedProjects.includes(project.name));
53+
9154
// include repos by glob if specified in config
9255
if (config.projects) {
93-
repos = includeReposByName(repos, config.projects);
56+
projects = projects.filter((project) => {
57+
return micromatch.isMatch(project.name, config.projects!);
58+
});
9459
}
95-
60+
9661
if (config.exclude && config.exclude.projects) {
97-
repos = excludeReposByName(repos, config.exclude.projects);
62+
projects = projects.filter((project) => {
63+
return !micromatch.isMatch(project.name, config.exclude!.projects!);
64+
});
9865
}
9966

100-
return repos;
67+
logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
68+
return projects;
10169
};
10270

103-
const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
71+
const fetchAllProjects = async (url: string): Promise<GerritProject[]> => {
10472
const projectsEndpoint = `${url}projects/`;
105-
let allProjects: GerritProjects = {};
73+
let allProjects: GerritProject[] = [];
10674
let start = 0; // Start offset for pagination
10775
let hasMoreProjects = true;
10876

@@ -119,8 +87,15 @@ const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
11987
const jsonText = text.replace(")]}'\n", ''); // Remove XSSI protection prefix
12088
const data: GerritProjects = JSON.parse(jsonText);
12189

122-
// Merge the current batch of projects with allProjects
123-
Object.assign(allProjects, data);
90+
// Add fetched projects to allProjects
91+
for (const [projectName, projectInfo] of Object.entries(data)) {
92+
allProjects.push({
93+
name: projectName,
94+
id: projectInfo.id,
95+
state: projectInfo.state,
96+
web_links: projectInfo.web_links
97+
})
98+
}
12499

125100
// Check if there are more projects to fetch
126101
hasMoreProjects = Object.values(data).some(

0 commit comments

Comments
 (0)