Add topics and exclude.topics to GitHub & GitLab config (#121)

brendan-kellam · web-flow · commit 4d358f94a2fd · 2024-12-11T14:17:57.000-08:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Added config support for filtering GitLab & GitHub repositories by topic. ([#121](https://github.com/sourcebot-dev/sourcebot/pull/121))
+
 ### Changed
 
 - Made language suggestions case insensitive. ([#124](https://github.com/sourcebot-dev/sourcebot/pull/124))
diff --git a/configs/filter.json b/configs/filter.json
@@ -63,5 +63,39 @@
                 ]
             }
         },
+        // Include all repos in my-org that have the topic
+        // "TypeScript" and do not have a topic that starts
+        // with "test-"
+        {
+            "type": "github",
+            "orgs": [
+                "my-org"
+            ],
+            "topics": [
+                "TypeScript"
+            ],
+            "exclude": {
+                "topics": [
+                    "test-**"
+                ]
+            }
+        },
+        // Include all repos in my-group that have the topic
+        // "TypeScript" and do not have a topic that starts
+        // with "test-"
+        {
+            "type": "gitlab",
+            "groups": [
+                "my-group"
+            ],
+            "topics": [
+                "TypeScript"
+            ],
+            "exclude": {
+                "topics": [
+                    "test-**"
+                ]
+            }
+        }
     ]
 }
diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts
@@ -3,7 +3,7 @@ import { GitHubConfig } from "./schemas/v2.js";
 import { createLogger } from "./logger.js";
 import { AppContext, GitRepository } from "./types.js";
 import path from 'path';
-import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js";
+import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, excludeReposByTopic, getTokenFromConfig, includeReposByTopic, marshalBool, measure } from "./utils.js";
 import micromatch from "micromatch";
 
 const logger = createLogger("GitHub");
@@ -21,6 +21,7 @@ type OctokitRepository = {
     subscribers_count?: number,
     forks_count?: number,
     archived?: boolean,
+    topics?: string[],
 }
 
 export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: AbortSignal, ctx: AppContext) => {
@@ -80,6 +81,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
                 isStale: false,
                 isFork: repo.fork,
                 isArchived: !!repo.archived,
+                topics: repo.topics ?? [],
                 gitConfigMetadata: {
                     'zoekt.web-url-type': 'github',
                     'zoekt.web-url': repo.html_url,
@@ -97,6 +99,10 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
             } satisfies GitRepository;
         });
 
+    if (config.topics) {
+        repos = includeReposByTopic(repos, config.topics, logger);
+    }
+
     if (config.exclude) {
         if (!!config.exclude.forks) {
             repos = excludeForkedRepos(repos, logger);
@@ -109,6 +115,10 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
         if (config.exclude.repos) {
             repos = excludeReposByName(repos, config.exclude.repos, logger);
         }
+
+        if (config.exclude.topics) {
+            repos = excludeReposByTopic(repos, config.exclude.topics, logger);
+        }
     }
 
     logger.debug(`Found ${repos.length} total repositories.`);
diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts
@@ -1,6 +1,6 @@
 import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
 import { GitLabConfig } from "./schemas/v2.js";
-import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js";
+import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, excludeReposByTopic, getTokenFromConfig, includeReposByTopic, marshalBool, measure } from "./utils.js";
 import { createLogger } from "./logger.js";
 import { AppContext, GitRepository } from "./types.js";
 import path from 'path';
@@ -98,6 +98,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
                 isStale: false,
                 isFork,
                 isArchived: project.archived,
+                topics: project.topics ?? [],
                 gitConfigMetadata: {
                     'zoekt.web-url-type': 'gitlab',
                     'zoekt.web-url': project.web_url,
@@ -113,6 +114,10 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
             } satisfies GitRepository;
         });
 
+    if (config.topics) {
+        repos = includeReposByTopic(repos, config.topics, logger);
+    }
+
     if (config.exclude) {
         if (!!config.exclude.forks) {
             repos = excludeForkedRepos(repos, logger);
@@ -125,6 +130,10 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
         if (config.exclude.projects) {
             repos = excludeReposByName(repos, config.exclude.projects, logger);
         }
+
+        if (config.exclude.topics) {
+            repos = excludeReposByTopic(repos, config.exclude.topics, logger);
+        }
     }
 
     logger.debug(`Found ${repos.length} total repositories.`);
diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts
@@ -54,6 +54,12 @@ export interface GitHubConfig {
    * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'.
    */
   repos?: string[];
+  /**
+   * List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.
+   *
+   * @minItems 1
+   */
+  topics?: [string, ...string[]];
   exclude?: {
     /**
      * Exclude forked repositories from syncing.
@@ -67,6 +73,10 @@ export interface GitHubConfig {
      * List of individual repositories to exclude from syncing. Glob patterns are supported.
      */
     repos?: string[];
+    /**
+     * List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.
+     */
+    topics?: string[];
   };
   revisions?: GitRevisions;
 }
@@ -119,6 +129,12 @@ export interface GitLabConfig {
    * List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/
    */
   projects?: string[];
+  /**
+   * List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.
+   *
+   * @minItems 1
+   */
+  topics?: [string, ...string[]];
   exclude?: {
     /**
      * Exclude forked projects from syncing.
@@ -132,6 +148,10 @@ export interface GitLabConfig {
      * List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/
      */
     projects?: string[];
+    /**
+     * List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.
+     */
+    topics?: string[];
   };
   revisions?: GitRevisions;
 }
diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts
@@ -8,6 +8,7 @@ interface BaseRepository {
     isFork?: boolean;
     isArchived?: boolean;
     codeHost?: string;
+    topics?: string[];
 }
 
 export interface GitRepository extends BaseRepository {
diff --git a/packages/backend/src/utils.test.ts b/packages/backend/src/utils.test.ts
@@ -1,5 +1,5 @@
 import { expect, test } from 'vitest';
-import { arraysEqualShallow, isRemotePath, excludeReposByName } from './utils';
+import { arraysEqualShallow, isRemotePath, excludeReposByName, includeReposByTopic, excludeReposByTopic } from './utils';
 import { Repository } from './types';
 
 const testNames: string[] = [
@@ -125,3 +125,135 @@ test('isRemotePath should return false for non HTTP paths', () => {
     expect(isRemotePath('')).toBe(false);
     expect(isRemotePath('   ')).toBe(false);
 });
+
+
+test('includeReposByTopic should return repos with matching topics', () => {
+    const repos = [
+        { id: '1', topics: ['javascript', 'typescript'] },
+        { id: '2', topics: ['python', 'django'] },
+        { id: '3', topics: ['typescript', 'react'] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = includeReposByTopic(repos, ['typescript']);
+    expect(result.length).toBe(2);
+    expect(result.map(r => r.id)).toEqual(['1', '3']);
+});
+
+test('includeReposByTopic should handle glob patterns in topic matching', () => {
+    const repos = [
+        { id: '1', topics: ['frontend-app', 'backend-app'] },
+        { id: '2', topics: ['mobile-app', 'web-app'] },
+        { id: '3', topics: ['desktop-app', 'cli-app'] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = includeReposByTopic(repos, ['*-app']);
+    expect(result.length).toBe(3);
+});
+
+test('includeReposByTopic should handle repos with no topics', () => {
+    const repos = [
+        { id: '1', topics: ['javascript'] },
+        { id: '2', topics: undefined },
+        { id: '3', topics: [] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = includeReposByTopic(repos, ['javascript']);
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe('1');
+});
+
+test('includeReposByTopic should return empty array when no repos match topics', () => {
+    const repos = [
+        { id: '1', topics: ['frontend'] },
+        { id: '2', topics: ['backend'] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = includeReposByTopic(repos, ['mobile']);
+    expect(result).toEqual([]);
+});
+
+
+test('excludeReposByTopic should exclude repos with matching topics', () => {
+    const repos = [
+        { id: '1', topics: ['javascript', 'typescript'] },
+        { id: '2', topics: ['python', 'django'] },
+        { id: '3', topics: ['typescript', 'react'] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = excludeReposByTopic(repos, ['typescript']);
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe('2');
+});
+
+test('excludeReposByTopic should handle glob patterns', () => {
+    const repos = [
+        { id: '1', topics: ['test-lib', 'test-app'] },
+        { id: '2', topics: ['prod-lib', 'prod-app'] },
+        { id: '3', topics: ['dev-tool'] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = excludeReposByTopic(repos, ['test-*']);
+    expect(result.length).toBe(2);
+    expect(result.map(r => r.id)).toEqual(['2', '3']);
+});
+
+test('excludeReposByTopic should handle multiple exclude patterns', () => {
+    const repos = [
+        { id: '1', topics: ['frontend', 'react'] },
+        { id: '2', topics: ['backend', 'node'] },
+        { id: '3', topics: ['mobile', 'react-native'] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = excludeReposByTopic(repos, ['*end', '*native']);
+    expect(result.length).toBe(0);
+});
+
+test('excludeReposByTopic should not exclude repos when no topics match', () => {
+    const repos = [
+        { id: '1', topics: ['frontend'] },
+        { id: '2', topics: ['backend'] },
+        { id: '3', topics: undefined }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = excludeReposByTopic(repos, ['mobile']);
+    expect(result.length).toBe(3);
+    expect(result.map(r => r.id)).toEqual(['1', '2', '3']);
+});
+
+test('excludeReposByTopic should handle empty exclude patterns array', () => {
+    const repos = [
+        { id: '1', topics: ['frontend'] },
+        { id: '2', topics: ['backend'] }
+    ].map(r => ({
+        ...createRepository(r.id),
+        ...r,
+    } satisfies Repository));
+
+    const result = excludeReposByTopic(repos, []);
+    expect(result.length).toBe(2);
+    expect(result).toEqual(repos);
+});
diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts
diff --git a/schemas/v2/index.json b/schemas/v2/index.json

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@ interface BaseRepository {`
`8`	`8`	`isFork?: boolean;`
`9`	`9`	`isArchived?: boolean;`
`10`	`10`	`codeHost?: string;`
	`11`	`+ topics?: string[];`
`11`	`12`	`}`
`12`	`13`
`13`	`14`	`export interface GitRepository extends BaseRepository {`