Skip to content

Commit 7c6adf1

Browse files
msukkaribrendan-kellam
msukkari
and
brendan-kellam
authored
Multi tenancy support in config syncer (#171)
* [wip] initial mt support in config syncer * Move logout button & profile picture into settings dropdown (#172) * update sync status properly and fix bug with multiple config in db case * make config path required in single tenant mode NOTE: deleting config/repos is currently not supported in multi tenancy case. Support for this will be added in a future PR --------- Co-authored-by: Brendan Kellam <bshizzle1234@gmail.com>
1 parent a5091fb commit 7c6adf1

File tree

10 files changed

+243
-53
lines changed

10 files changed

+243
-53
lines changed

‎package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,16 @@
66
"scripts": {
77
"build": "yarn workspaces run build",
88
"test": "yarn workspaces run test",
9-
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
10-
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
9+
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
10+
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
1111
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
1212
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
1313
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
1414
"dev:web": "yarn workspace @sourcebot/web dev",
1515
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
16-
1716
},
1817
"devDependencies": {
18+
"cross-env": "^7.0.3",
1919
"npm-run-all": "^4.1.5"
2020
}
2121
}

‎packages/backend/src/config.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { SourcebotConfigurationSchema } from "./schemas/v2.js";
77
import { AppContext } from "./types.js";
88
import { getTokenFromConfig, isRemotePath, marshalBool } from "./utils.js";
99

10-
export const syncConfig = async (configPath: string, db: PrismaClient, signal: AbortSignal, ctx: AppContext) => {
10+
export const fetchConfigFromPath = async (configPath: string, signal: AbortSignal) => {
1111
const configContent = await (async () => {
1212
if (isRemotePath(configPath)) {
1313
const response = await fetch(configPath, {
@@ -25,9 +25,11 @@ export const syncConfig = async (configPath: string, db: PrismaClient, signal: A
2525
}
2626
})();
2727

28-
// @todo: we should validate the configuration file's structure here.
2928
const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfigurationSchema;
29+
return config;
30+
}
3031

32+
export const syncConfig = async (config: SourcebotConfigurationSchema, db: PrismaClient, signal: AbortSignal, ctx: AppContext) => {
3133
for (const repoConfig of config.repos ?? []) {
3234
switch (repoConfig.type) {
3335
case 'github': {

‎packages/backend/src/constants.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ export const DEFAULT_SETTINGS: Settings = {
99
reindexIntervalMs: 1000 * 60,
1010
resyncIntervalMs: 1000 * 60 * 60 * 24, // 1 day in milliseconds
1111
indexConcurrencyMultiple: 3,
12+
configSyncConcurrencyMultiple: 3,
1213
}

‎packages/backend/src/environment.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import dotenv from 'dotenv';
22

3-
export const getEnv = (env: string | undefined, defaultValue?: string) => {
3+
export const getEnv = (env: string | undefined, defaultValue?: string, required?: boolean) => {
4+
if (required && !env && !defaultValue) {
5+
throw new Error(`Missing required environment variable`);
6+
}
7+
48
return env ?? defaultValue;
59
}
610

@@ -15,6 +19,8 @@ dotenv.config({
1519
path: './.env',
1620
});
1721

22+
23+
export const SOURCEBOT_TENANT_MODE = getEnv(process.env.SOURCEBOT_TENANT_MODE, undefined, true);
1824
export const SOURCEBOT_LOG_LEVEL = getEnv(process.env.SOURCEBOT_LOG_LEVEL, 'info')!;
1925
export const SOURCEBOT_TELEMETRY_DISABLED = getEnvBoolean(process.env.SOURCEBOT_TELEMETRY_DISABLED, false)!;
2026
export const SOURCEBOT_INSTALL_ID = getEnv(process.env.SOURCEBOT_INSTALL_ID, 'unknown')!;

‎packages/backend/src/index.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { isRemotePath } from "./utils.js";
66
import { AppContext } from "./types.js";
77
import { main } from "./main.js"
88
import { PrismaClient } from "@sourcebot/db";
9+
import { SOURCEBOT_TENANT_MODE } from "./environment.js";
910

1011

1112
const parser = new ArgumentParser({
@@ -19,7 +20,7 @@ type Arguments = {
1920

2021
parser.add_argument("--configPath", {
2122
help: "Path to config file",
22-
required: true,
23+
required: SOURCEBOT_TENANT_MODE === "single",
2324
});
2425

2526
parser.add_argument("--cacheDir", {
@@ -28,8 +29,8 @@ parser.add_argument("--cacheDir", {
2829
});
2930
const args = parser.parse_args() as Arguments;
3031

31-
if (!isRemotePath(args.configPath) && !existsSync(args.configPath)) {
32-
console.error(`Config file ${args.configPath} does not exist`);
32+
if (SOURCEBOT_TENANT_MODE === "single" && !isRemotePath(args.configPath) && !existsSync(args.configPath)) {
33+
console.error(`Config file ${args.configPath} does not exist, and is required in single tenant mode`);
3334
process.exit(1);
3435
}
3536

‎packages/backend/src/main.ts‎

Lines changed: 160 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import { PrismaClient, Repo, RepoIndexingStatus } from '@sourcebot/db';
1+
import { ConfigSyncStatus, PrismaClient, Repo, Config, RepoIndexingStatus, Prisma } from '@sourcebot/db';
22
import { existsSync, watch } from 'fs';
3-
import { syncConfig } from "./config.js";
3+
import { fetchConfigFromPath, syncConfig } from "./config.js";
44
import { cloneRepository, fetchRepository } from "./git.js";
55
import { createLogger } from "./logger.js";
66
import { captureEvent } from "./posthog.js";
@@ -11,6 +11,8 @@ import { DEFAULT_SETTINGS } from './constants.js';
1111
import { Queue, Worker, Job } from 'bullmq';
1212
import { Redis } from 'ioredis';
1313
import * as os from 'os';
14+
import { SOURCEBOT_TENANT_MODE } from './environment.js';
15+
import { SourcebotConfigurationSchema } from './schemas/v2.js';
1416

1517
const logger = createLogger('main');
1618

@@ -56,6 +58,23 @@ const syncGitRepository = async (repo: Repo, ctx: AppContext) => {
5658
}
5759
}
5860

61+
async function addConfigsToQueue(db: PrismaClient, queue: Queue, configs: Config[]) {
62+
for (const config of configs) {
63+
await db.$transaction(async (tx) => {
64+
await tx.config.update({
65+
where: { id: config.id },
66+
data: { syncStatus: ConfigSyncStatus.IN_SYNC_QUEUE },
67+
});
68+
69+
// Add the job to the queue
70+
await queue.add('configSyncJob', config);
71+
logger.info(`Added job to queue for config ${config.id}`);
72+
}).catch((err: unknown) => {
73+
logger.error(`Failed to add job to queue for config ${config.id}: ${err}`);
74+
});
75+
}
76+
}
77+
5978
async function addReposToQueue(db: PrismaClient, queue: Queue, repos: Repo[]) {
6079
for (const repo of repos) {
6180
await db.$transaction(async (tx) => {
@@ -67,7 +86,7 @@ async function addReposToQueue(db: PrismaClient, queue: Queue, repos: Repo[]) {
6786
// Add the job to the queue
6887
await queue.add('indexJob', repo);
6988
logger.info(`Added job to queue for repo ${repo.id}`);
70-
}).catch((err) => {
89+
}).catch((err: unknown) => {
7190
logger.error(`Failed to add job to queue for repo ${repo.id}: ${err}`);
7291
});
7392
}
@@ -76,66 +95,166 @@ async function addReposToQueue(db: PrismaClient, queue: Queue, repos: Repo[]) {
7695
export const main = async (db: PrismaClient, context: AppContext) => {
7796
let abortController = new AbortController();
7897
let isSyncing = false;
79-
const _syncConfig = async () => {
80-
if (isSyncing) {
81-
abortController.abort();
82-
abortController = new AbortController();
83-
}
98+
const _syncConfig = async (dbConfig?: Config | undefined) => {
8499

85-
logger.info(`Syncing configuration file ${context.configPath} ...`);
86-
isSyncing = true;
100+
// Fetch config object and update syncing status
101+
let config: SourcebotConfigurationSchema;
102+
switch (SOURCEBOT_TENANT_MODE) {
103+
case 'single':
104+
logger.info(`Syncing configuration file ${context.configPath} ...`);
87105

106+
if (isSyncing) {
107+
abortController.abort();
108+
abortController = new AbortController();
109+
}
110+
config = await fetchConfigFromPath(context.configPath, abortController.signal);
111+
isSyncing = true;
112+
break;
113+
case 'multi':
114+
if(!dbConfig) {
115+
throw new Error('config object is required in multi tenant mode');
116+
}
117+
config = dbConfig.data as SourcebotConfigurationSchema
118+
db.config.update({
119+
where: {
120+
id: dbConfig.id,
121+
},
122+
data: {
123+
syncStatus: ConfigSyncStatus.SYNCING,
124+
}
125+
})
126+
break;
127+
default:
128+
throw new Error(`Invalid SOURCEBOT_TENANT_MODE: ${SOURCEBOT_TENANT_MODE}`);
129+
}
130+
131+
// Attempt to sync the config, handle failure cases
88132
try {
89-
const { durationMs } = await measure(() => syncConfig(context.configPath, db, abortController.signal, context))
90-
logger.info(`Synced configuration file ${context.configPath} in ${durationMs / 1000}s`);
133+
const { durationMs } = await measure(() => syncConfig(config, db, abortController.signal, context))
134+
logger.info(`Synced configuration in ${durationMs / 1000}s`);
91135
isSyncing = false;
92136
} catch (err: any) {
93-
if (err.name === "AbortError") {
94-
// @note: If we're aborting, we don't want to set isSyncing to false
95-
// since it implies another sync is in progress.
96-
} else {
97-
isSyncing = false;
98-
logger.error(`Failed to sync configuration file ${context.configPath} with error:`);
99-
console.log(err);
137+
switch(SOURCEBOT_TENANT_MODE) {
138+
case 'single':
139+
if (err.name === "AbortError") {
140+
// @note: If we're aborting, we don't want to set isSyncing to false
141+
// since it implies another sync is in progress.
142+
} else {
143+
isSyncing = false;
144+
logger.error(`Failed to sync configuration file with error:`);
145+
console.log(err);
146+
}
147+
break;
148+
case 'multi':
149+
if (dbConfig) {
150+
await db.config.update({
151+
where: {
152+
id: dbConfig.id,
153+
},
154+
data: {
155+
syncStatus: ConfigSyncStatus.FAILED,
156+
}
157+
})
158+
logger.error(`Failed to sync configuration ${dbConfig.id} with error: ${err}`);
159+
} else {
160+
logger.error(`DB config undefined. Failed to sync configuration with error: ${err}`);
161+
}
162+
break;
163+
default:
164+
throw new Error(`Invalid SOURCEBOT_TENANT_MODE: ${SOURCEBOT_TENANT_MODE}`);
100165
}
101166
}
102167
}
103168

104-
// Re-sync on file changes if the config file is local
105-
if (!isRemotePath(context.configPath)) {
106-
watch(context.configPath, () => {
107-
logger.info(`Config file ${context.configPath} changed. Re-syncing...`);
108-
_syncConfig();
109-
});
110-
}
111-
112-
// Re-sync at a fixed interval
113-
setInterval(() => {
114-
_syncConfig();
115-
}, DEFAULT_SETTINGS.resyncIntervalMs);
116-
117-
// Sync immediately on startup
118-
await _syncConfig();
119-
169+
/////////////////////////////
170+
// Init Redis
171+
/////////////////////////////
120172
const redis = new Redis({
121173
host: 'localhost',
122174
port: 6379,
123175
maxRetriesPerRequest: null
124176
});
125177
redis.ping().then(() => {
126178
logger.info('Connected to redis');
127-
}).catch((err) => {
179+
}).catch((err: unknown) => {
128180
logger.error('Failed to connect to redis');
129181
console.error(err);
130182
process.exit(1);
131183
});
132184

185+
/////////////////////////////
186+
// Setup config sync watchers
187+
/////////////////////////////
188+
switch (SOURCEBOT_TENANT_MODE) {
189+
case 'single':
190+
// Re-sync on file changes if the config file is local
191+
if (!isRemotePath(context.configPath)) {
192+
watch(context.configPath, () => {
193+
logger.info(`Config file ${context.configPath} changed. Re-syncing...`);
194+
_syncConfig();
195+
});
196+
}
197+
198+
// Re-sync at a fixed interval
199+
setInterval(() => {
200+
_syncConfig();
201+
}, DEFAULT_SETTINGS.resyncIntervalMs);
202+
203+
// Sync immediately on startup
204+
await _syncConfig();
205+
break;
206+
case 'multi':
207+
// Setup config sync queue and workers
208+
const configSyncQueue = new Queue('configSyncQueue');
209+
const numCores = os.cpus().length;
210+
const numWorkers = numCores * DEFAULT_SETTINGS.configSyncConcurrencyMultiple;
211+
logger.info(`Detected ${numCores} cores. Setting config sync max concurrency to ${numWorkers}`);
212+
const configSyncWorker = new Worker('configSyncQueue', async (job: Job) => {
213+
const config = job.data as Config;
214+
await _syncConfig(config);
215+
}, { connection: redis, concurrency: numWorkers });
216+
configSyncWorker.on('completed', async (job: Job) => {
217+
logger.info(`Config sync job ${job.id} completed`);
218+
219+
const config = job.data as Config;
220+
await db.config.update({
221+
where: {
222+
id: config.id,
223+
},
224+
data: {
225+
syncStatus: ConfigSyncStatus.SYNCED,
226+
}
227+
})
228+
});
229+
configSyncWorker.on('failed', (job: Job | undefined, err: unknown) => {
230+
logger.info(`Config sync job failed with error: ${err}`);
231+
});
232+
233+
setInterval(async () => {
234+
const configs = await db.config.findMany({
235+
where: {
236+
syncStatus: ConfigSyncStatus.SYNC_NEEDED,
237+
}
238+
});
239+
240+
logger.info(`Found ${configs.length} configs to sync...`);
241+
addConfigsToQueue(db, configSyncQueue, configs);
242+
}, 1000);
243+
break;
244+
default:
245+
throw new Error(`Invalid SOURCEBOT_TENANT_MODE: ${SOURCEBOT_TENANT_MODE}`);
246+
}
247+
248+
249+
/////////////////////////
250+
// Setup repo indexing
251+
/////////////////////////
133252
const indexQueue = new Queue('indexQueue');
134253

135254
const numCores = os.cpus().length;
136255
const numWorkers = numCores * DEFAULT_SETTINGS.indexConcurrencyMultiple;
137-
logger.info(`Detected ${numCores} cores. Setting max concurrency to ${numWorkers}`);
138-
const worker = new Worker('indexQueue', async (job) => {
256+
logger.info(`Detected ${numCores} cores. Setting repo index max concurrency to ${numWorkers}`);
257+
const worker = new Worker('indexQueue', async (job: Job) => {
139258
const repo = job.data as Repo;
140259

141260
let indexDuration_s: number | undefined;
@@ -166,10 +285,10 @@ export const main = async (db: PrismaClient, context: AppContext) => {
166285
});
167286
}, { connection: redis, concurrency: numWorkers });
168287

169-
worker.on('completed', (job) => {
288+
worker.on('completed', (job: Job) => {
170289
logger.info(`Job ${job.id} completed`);
171290
});
172-
worker.on('failed', async (job: Job | undefined, err) => {
291+
worker.on('failed', async (job: Job | undefined, err: unknown) => {
173292
logger.info(`Job failed with error: ${err}`);
174293
if (job) {
175294
await db.repo.update({
@@ -183,6 +302,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
183302
}
184303
});
185304

305+
// Repo indexing loop
186306
while (true) {
187307
const thresholdDate = new Date(Date.now() - DEFAULT_SETTINGS.reindexIntervalMs);
188308
const repos = await db.repo.findMany({

‎packages/backend/src/types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ export type Settings = {
7878
* The multiple of the number of CPUs to use for indexing.
7979
*/
8080
indexConcurrencyMultiple: number;
81+
/**
82+
* The multiple of the number of CPUs to use for syncing the configuration.
83+
*/
84+
configSyncConcurrencyMultiple: number;
8185
}
8286

8387
// @see : https://stackoverflow.com/a/61132308

‎packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-- CreateTable
2+
CREATE TABLE "Config" (
3+
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
4+
"data" JSONB NOT NULL,
5+
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
6+
"updatedAt" DATETIME NOT NULL,
7+
"syncedAt" DATETIME,
8+
"syncStatus" TEXT NOT NULL DEFAULT 'SYNC_NEEDED',
9+
"orgId" INTEGER NOT NULL,
10+
CONSTRAINT "Config_orgId_fkey" FOREIGN KEY ("orgId") REFERENCES "Org" ("id") ON DELETE CASCADE ON UPDATE CASCADE
11+
);

0 commit comments

Comments
 (0)