Local directory support

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Local directory indexing support. ([#56](https://github.com/sourcebot-dev/sourcebot/pull/56))
+
 ## [2.2.0] - 2024-10-30
 
 ### Added

Dockerfile

@@ -76,6 +76,7 @@ COPY --from=zoekt-builder \
     /cmd/zoekt-mirror-gitlab \
     /cmd/zoekt-mirror-gerrit \
     /cmd/zoekt-webserver \
+    /cmd/zoekt-index \
     /usr/local/bin/
 
 # Configure the webapp

README.md

@@ -267,6 +267,37 @@ docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/s
 
 If you're using a self-hosted GitLab or GitHub instance with a custom domain, you can specify the domain in your config file. See [configs/self-hosted.json](configs/self-hosted.json) for examples.
 
+## Searching a local directory
+
+Local directories can be searched by using the `local` type in your config file:
+
+```jsonc
+{
+    "$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "local",
+            "path": "/repos/my-repo",
+            // re-index files when a change is detected
+            "watch": true,
+            "exclude": {
+                // exclude paths from being indexed
+                "paths": [
+                    "node_modules",
+                    "build"
+                ]
+            }
+        }
+    ]
+}
+```
+
+You'll need to mount the directory as a volume when running Sourcebot:
+
+<pre>
+docker run <b>-v /path/to/my-repo:/repos/my-repo</b> /* additional args */ ghcr.io/sourcebot-dev/sourcebot:latest
+</pre>
+
 ## Build from source
 >[!NOTE]
 > Building from source is only required if you'd like to contribute. The recommended way to use Sourcebot is to use the [pre-built docker image](https://github.com/sourcebot-dev/sourcebot/pkgs/container/sourcebot).

configs/auth.json

@@ -17,6 +17,13 @@
                 "my-group"
             ]
         },
+        {
+            "type": "gitea",
+            "token": "gitea-token",
+            "orgs": [
+                "my-org"
+            ]
+        },
 
         // You can also store the token in a environment variable and then
         // references it from the config.
@@ -34,6 +41,15 @@
             "groups": [
                 "my-group"
             ]
+        },
+        {
+            "type": "gitea",
+            "token": {
+                "env": "GITEA_TOKEN_ENV_VAR"
+            },
+            "orgs": [
+                "my-org"
+            ]
         }
     ]
 }

configs/basic.json

@@ -37,6 +37,28 @@
             "projects": [
                 "my-group/project1"
             ]
+        },
+        // From Gitea, include:
+        // - all public repos owned by user `my-user`
+        // - all public repos owned by organization `my-org`
+        // - repo `my-org/my-repo`
+        {
+            "type": "gitea",
+            "token": "my-token",
+            "users": [
+                "my-user"
+            ],
+            "orgs": [
+                "my-org"
+            ],
+            "repos": [
+                "my-org/my-repo"
+            ]
+        },
+        // Index a local repository
+        {
+            "type": "local",
+            "path": "/path/to/local/repo"
         }
     ]
 }

configs/filter.json

@@ -37,6 +37,25 @@
                     "my-group/project2"
                 ]
             }
-        }
+        },
+
+        // Include all repos in my-org, except:
+        // - repo1 & repo2
+        // - repos that are archived or forks
+        {
+            "type": "gitea",
+            "token": "my-token",
+            "orgs": [
+                "my-org"
+            ],
+            "exclude": {
+                "archived": true,
+                "forks": true,
+                "repos": [
+                    "my-org/repo1",
+                    "my-org/repo2"
+                ]
+            }
+        },
     ]
 }

configs/local-repo.json

@@ -0,0 +1,32 @@
+{
+    "$schema": "../schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "local",
+            "path": "/path/to/local/repo"
+        },
+        // Relative paths are relative to the config file
+        {
+            "type": "local",
+            "path": "../../relative/path/to/local/repo"
+        },
+        // File watcher can be disabled (enabled by default)
+        {
+            "type": "local",
+            "path": "/path/to/local/repo",
+            "watch": false
+        },
+        // Exclude paths can be specified
+        {
+            "type": "local",
+            "path": "/path/to/local/repo",
+            "exclude": {
+                "paths": [
+                    ".git",
+                    "node_modules",
+                    "dist"
+                ]
+            }
+        }
+    ]
+}

configs/self-hosted.json

@@ -14,6 +14,13 @@
             "groups": [
                 "my-group"
             ]
+        },
+        {
+            "type": "gitea",
+            "url": "https://gitea.example.com",
+            "orgs": [
+                "my-org-name"
+            ]
         }
     ]
 }

packages/backend/src/db.ts

@@ -14,7 +14,7 @@ export const loadDB = async (ctx: AppContext): Promise<Database> => {
     const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, { repos: {} });
     return db;
 }
-export const updateRepository = async (repoId: string, data: Partial<Repository>, db: Database) => {
+export const updateRepository = async (repoId: string, data: Repository, db: Database) => {
     db.data.repos[repoId] = {
         ...db.data.repos[repoId],
         ...data,

packages/backend/src/git.ts

@@ -1,11 +1,11 @@
-import { Repository } from './types.js';
+import { GitRepository } from './types.js';
 import { simpleGit, SimpleGitProgressEvent } from 'simple-git';
 import { existsSync } from 'fs';
 import { createLogger } from './logger.js';
 
 const logger = createLogger('git');
 
-export const cloneRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
+export const cloneRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
     if (existsSync(repo.path)) {
         logger.warn(`${repo.id} already exists. Skipping clone.`)
         return;
@@ -34,7 +34,7 @@ export const cloneRepository = async (repo: Repository, onProgress?: (event: Sim
 }
 
 
-export const fetchRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
+export const fetchRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
     const git = simpleGit({
         progress: onProgress,
     });

packages/backend/src/gitea.ts

@@ -1,7 +1,7 @@
 import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
 import { GiteaConfig } from './schemas/v2.js';
 import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js';
-import { AppContext, Repository } from './types.js';
+import { AppContext, GitRepository } from './types.js';
 import fetch from 'cross-fetch';
 import { createLogger } from './logger.js';
 import path from 'path';
@@ -33,7 +33,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
         allRepos = allRepos.concat(_repos);
     }
 
-    let repos: Repository[] = allRepos
+    let repos: GitRepository[] = allRepos
         .map((repo) => {
             const hostname = config.url ? new URL(config.url).hostname : 'gitea.com';
             const repoId = `${hostname}/${repo.full_name!}`;
@@ -45,6 +45,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
             }
 
             return {
+                vcs: 'git',
                 name: repo.full_name!,
                 id: repoId,
                 cloneUrl: cloneUrl.toString(),
@@ -60,7 +61,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
                     'zoekt.fork': marshalBool(repo.fork!),
                     'zoekt.public': marshalBool(repo.internal === false && repo.private === false),
                 }
-            } satisfies Repository;
+            } satisfies GitRepository;
         });
 
     if (config.exclude) {

packages/backend/src/github.ts

@@ -1,7 +1,7 @@
 import { Octokit } from "@octokit/rest";
 import { GitHubConfig } from "./schemas/v2.js";
 import { createLogger } from "./logger.js";
-import { AppContext, Repository } from "./types.js";
+import { AppContext, GitRepository } from "./types.js";
 import path from 'path';
 import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool } from "./utils.js";
 
@@ -50,7 +50,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
     }
 
     // Marshall results to our type
-    let repos: Repository[] = allRepos
+    let repos: GitRepository[] = allRepos
         .filter((repo) => {
             if (!repo.clone_url) {
                 logger.warn(`Repository ${repo.name} missing property 'clone_url'. Excluding.`)
@@ -69,6 +69,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
             }
 
             return {
+                vcs: 'git',
                 name: repo.full_name,
                 id: repoId,
                 cloneUrl: cloneUrl.toString(),
@@ -88,7 +89,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
                     'zoekt.fork': marshalBool(repo.fork),
                     'zoekt.public': marshalBool(repo.private === false)
                 }
-            } satisfies Repository;
+            } satisfies GitRepository;
         });
 
     if (config.exclude) {

packages/backend/src/gitlab.ts

@@ -2,7 +2,7 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
 import { GitLabConfig } from "./schemas/v2.js";
 import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js";
 import { createLogger } from "./logger.js";
-import { AppContext, Repository } from "./types.js";
+import { AppContext, GitRepository } from "./types.js";
 import path from 'path';
 
 const logger = createLogger("GitLab");
@@ -59,7 +59,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
         allProjects = allProjects.concat(_projects);
     }
 
-    let repos: Repository[] = allProjects
+    let repos: GitRepository[] = allProjects
         .map((project) => {
             const hostname = config.url ? new URL(config.url).hostname : "gitlab.com";
             const repoId = `${hostname}/${project.path_with_namespace}`;
@@ -73,6 +73,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
             }
 
             return {
+                vcs: 'git',
                 name: project.path_with_namespace,
                 id: repoId,
                 cloneUrl: cloneUrl.toString(),
@@ -90,7 +91,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
                     'zoekt.fork': marshalBool(isFork),
                     'zoekt.public': marshalBool(project.visibility === 'public'),
                 }
-            } satisfies Repository;
+            } satisfies GitRepository;
         });
 
     if (config.exclude) {