Skip to content

Commit 7966c14

Browse files
brendan-kellambrendan-kellam
authored
Local directory support (#56)
1 parent 3b8e920 commit 7966c14

File tree

23 files changed

+438
-98
lines changed

23 files changed

+438
-98
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
### Added
11+
12+
- Local directory indexing support. ([#56](https://github.com/sourcebot-dev/sourcebot/pull/56))
13+
1014
## [2.2.0] - 2024-10-30
1115

1216
### Added

‎Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ COPY --from=zoekt-builder \
7676
/cmd/zoekt-mirror-gitlab \
7777
/cmd/zoekt-mirror-gerrit \
7878
/cmd/zoekt-webserver \
79+
/cmd/zoekt-index \
7980
/usr/local/bin/
8081

8182
# Configure the webapp

‎README.md

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,37 @@ docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/s
267267

268268
If you're using a self-hosted GitLab or GitHub instance with a custom domain, you can specify the domain in your config file. See [configs/self-hosted.json](configs/self-hosted.json) for examples.
269269

270+
## Searching a local directory
271+
272+
Local directories can be searched by using the `local` type in your config file:
273+
274+
```jsonc
275+
{
276+
"$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v2/index.json",
277+
"repos": [
278+
{
279+
"type": "local",
280+
"path": "/repos/my-repo",
281+
// re-index files when a change is detected
282+
"watch": true,
283+
"exclude": {
284+
// exclude paths from being indexed
285+
"paths": [
286+
"node_modules",
287+
"build"
288+
]
289+
}
290+
}
291+
]
292+
}
293+
```
294+
295+
You'll need to mount the directory as a volume when running Sourcebot:
296+
297+
<pre>
298+
docker run <b>-v /path/to/my-repo:/repos/my-repo</b> /* additional args */ ghcr.io/sourcebot-dev/sourcebot:latest
299+
</pre>
300+
270301
## Build from source
271302
>[!NOTE]
272303
> Building from source is only required if you'd like to contribute. The recommended way to use Sourcebot is to use the [pre-built docker image](https://github.com/sourcebot-dev/sourcebot/pkgs/container/sourcebot).

‎configs/auth.json

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@
1717
"my-group"
1818
]
1919
},
20+
{
21+
"type": "gitea",
22+
"token": "gitea-token",
23+
"orgs": [
24+
"my-org"
25+
]
26+
},
2027

2128
// You can also store the token in a environment variable and then
2229
// references it from the config.
@@ -34,6 +41,15 @@
3441
"groups": [
3542
"my-group"
3643
]
44+
},
45+
{
46+
"type": "gitea",
47+
"token": {
48+
"env": "GITEA_TOKEN_ENV_VAR"
49+
},
50+
"orgs": [
51+
"my-org"
52+
]
3753
}
3854
]
3955
}

‎configs/basic.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,28 @@
3737
"projects": [
3838
"my-group/project1"
3939
]
40+
},
41+
// From Gitea, include:
42+
// - all public repos owned by user `my-user`
43+
// - all public repos owned by organization `my-org`
44+
// - repo `my-org/my-repo`
45+
{
46+
"type": "gitea",
47+
"token": "my-token",
48+
"users": [
49+
"my-user"
50+
],
51+
"orgs": [
52+
"my-org"
53+
],
54+
"repos": [
55+
"my-org/my-repo"
56+
]
57+
},
58+
// Index a local repository
59+
{
60+
"type": "local",
61+
"path": "/path/to/local/repo"
4062
}
4163
]
4264
}

‎configs/filter.json

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,25 @@
3737
"my-group/project2"
3838
]
3939
}
40-
}
40+
},
41+
42+
// Include all repos in my-org, except:
43+
// - repo1 & repo2
44+
// - repos that are archived or forks
45+
{
46+
"type": "gitea",
47+
"token": "my-token",
48+
"orgs": [
49+
"my-org"
50+
],
51+
"exclude": {
52+
"archived": true,
53+
"forks": true,
54+
"repos": [
55+
"my-org/repo1",
56+
"my-org/repo2"
57+
]
58+
}
59+
},
4160
]
4261
}

‎configs/local-repo.json

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"$schema": "../schemas/v2/index.json",
3+
"repos": [
4+
{
5+
"type": "local",
6+
"path": "/path/to/local/repo"
7+
},
8+
// Relative paths are relative to the config file
9+
{
10+
"type": "local",
11+
"path": "../../relative/path/to/local/repo"
12+
},
13+
// File watcher can be disabled (enabled by default)
14+
{
15+
"type": "local",
16+
"path": "/path/to/local/repo",
17+
"watch": false
18+
},
19+
// Exclude paths can be specified
20+
{
21+
"type": "local",
22+
"path": "/path/to/local/repo",
23+
"exclude": {
24+
"paths": [
25+
".git",
26+
"node_modules",
27+
"dist"
28+
]
29+
}
30+
}
31+
]
32+
}

‎configs/self-hosted.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@
1414
"groups": [
1515
"my-group"
1616
]
17+
},
18+
{
19+
"type": "gitea",
20+
"url": "https://gitea.example.com",
21+
"orgs": [
22+
"my-org-name"
23+
]
1724
}
1825
]
1926
}

‎packages/backend/src/db.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export const loadDB = async (ctx: AppContext): Promise<Database> => {
1414
const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, { repos: {} });
1515
return db;
1616
}
17-
export const updateRepository = async (repoId: string, data: Partial<Repository>, db: Database) => {
17+
export const updateRepository = async (repoId: string, data: Repository, db: Database) => {
1818
db.data.repos[repoId] = {
1919
...db.data.repos[repoId],
2020
...data,

‎packages/backend/src/git.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
import { Repository } from './types.js';
1+
import { GitRepository } from './types.js';
22
import { simpleGit, SimpleGitProgressEvent } from 'simple-git';
33
import { existsSync } from 'fs';
44
import { createLogger } from './logger.js';
55

66
const logger = createLogger('git');
77

8-
export const cloneRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
8+
export const cloneRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
99
if (existsSync(repo.path)) {
1010
logger.warn(`${repo.id} already exists. Skipping clone.`)
1111
return;
@@ -34,7 +34,7 @@ export const cloneRepository = async (repo: Repository, onProgress?: (event: Sim
3434
}
3535

3636

37-
export const fetchRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
37+
export const fetchRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
3838
const git = simpleGit({
3939
progress: onProgress,
4040
});

‎packages/backend/src/gitea.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
22
import { GiteaConfig } from './schemas/v2.js';
33
import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js';
4-
import { AppContext, Repository } from './types.js';
4+
import { AppContext, GitRepository } from './types.js';
55
import fetch from 'cross-fetch';
66
import { createLogger } from './logger.js';
77
import path from 'path';
@@ -33,7 +33,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
3333
allRepos = allRepos.concat(_repos);
3434
}
3535

36-
let repos: Repository[] = allRepos
36+
let repos: GitRepository[] = allRepos
3737
.map((repo) => {
3838
const hostname = config.url ? new URL(config.url).hostname : 'gitea.com';
3939
const repoId = `${hostname}/${repo.full_name!}`;
@@ -45,6 +45,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
4545
}
4646

4747
return {
48+
vcs: 'git',
4849
name: repo.full_name!,
4950
id: repoId,
5051
cloneUrl: cloneUrl.toString(),
@@ -60,7 +61,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
6061
'zoekt.fork': marshalBool(repo.fork!),
6162
'zoekt.public': marshalBool(repo.internal === false && repo.private === false),
6263
}
63-
} satisfies Repository;
64+
} satisfies GitRepository;
6465
});
6566

6667
if (config.exclude) {

‎packages/backend/src/github.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Octokit } from "@octokit/rest";
22
import { GitHubConfig } from "./schemas/v2.js";
33
import { createLogger } from "./logger.js";
4-
import { AppContext, Repository } from "./types.js";
4+
import { AppContext, GitRepository } from "./types.js";
55
import path from 'path';
66
import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool } from "./utils.js";
77

@@ -50,7 +50,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
5050
}
5151

5252
// Marshall results to our type
53-
let repos: Repository[] = allRepos
53+
let repos: GitRepository[] = allRepos
5454
.filter((repo) => {
5555
if (!repo.clone_url) {
5656
logger.warn(`Repository ${repo.name} missing property 'clone_url'. Excluding.`)
@@ -69,6 +69,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
6969
}
7070

7171
return {
72+
vcs: 'git',
7273
name: repo.full_name,
7374
id: repoId,
7475
cloneUrl: cloneUrl.toString(),
@@ -88,7 +89,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
8889
'zoekt.fork': marshalBool(repo.fork),
8990
'zoekt.public': marshalBool(repo.private === false)
9091
}
91-
} satisfies Repository;
92+
} satisfies GitRepository;
9293
});
9394

9495
if (config.exclude) {

‎packages/backend/src/gitlab.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
22
import { GitLabConfig } from "./schemas/v2.js";
33
import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js";
44
import { createLogger } from "./logger.js";
5-
import { AppContext, Repository } from "./types.js";
5+
import { AppContext, GitRepository } from "./types.js";
66
import path from 'path';
77

88
const logger = createLogger("GitLab");
@@ -59,7 +59,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
5959
allProjects = allProjects.concat(_projects);
6060
}
6161

62-
let repos: Repository[] = allProjects
62+
let repos: GitRepository[] = allProjects
6363
.map((project) => {
6464
const hostname = config.url ? new URL(config.url).hostname : "gitlab.com";
6565
const repoId = `${hostname}/${project.path_with_namespace}`;
@@ -73,6 +73,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
7373
}
7474

7575
return {
76+
vcs: 'git',
7677
name: project.path_with_namespace,
7778
id: repoId,
7879
cloneUrl: cloneUrl.toString(),
@@ -90,7 +91,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
9091
'zoekt.fork': marshalBool(isFork),
9192
'zoekt.public': marshalBool(project.visibility === 'public'),
9293
}
93-
} satisfies Repository;
94+
} satisfies GitRepository;
9495
});
9596

9697
if (config.exclude) {

0 commit comments

Comments
 (0)