add retries for 429 issues

Makefile

@@ -28,4 +28,9 @@ clean:
 		packages/crypto/dist \
 		.sourcebot
 
+soft-reset:
+	rm -rf .sourcebot
+	redis-cli FLUSHALL
+
+
 .PHONY: bin

demo-site-config.json

@@ -18,7 +18,7 @@
             },
             "repos": [
                 "torvalds/linux",
-                "pytorch/pytorch",
+                "pytorch/pytorch", 
                 "commaai/openpilot",
                 "ggerganov/whisper.cpp",
                 "ggerganov/llama.cpp",
@@ -42,7 +42,6 @@
                 "TheAlgorithms/Python",
                 "trekhleb/javascript-algorithms",
                 "tensorflow/tensorflow",
-                "torvalds/linux",
                 "getify/You-Dont-Know-JS",
                 "CyC2018/CS-Notes",
                 "ohmyzsh/ohmyzsh",
@@ -106,10 +105,8 @@
                 "Hack-with-Github/Awesome-Hacking",
                 "nvbn/thefuck",
                 "mtdvio/every-programmer-should-know",
-                "pytorch/pytorch",
                 "storybookjs/storybook",
                 "neovim/neovim",
-                "tailwindlabs/tailwindcss",
                 "microsoft/Web-Dev-For-Beginners",
                 "django/django",
                 "florinpop17/app-ideas",
@@ -153,7 +150,6 @@
                 "fighting41love/funNLP",
                 "vitejs/vite",
                 "thedaviddias/Front-End-Checklist",
-                "ggerganov/llama.cpp",
                 "coder/code-server",
                 "moby/moby",
                 "CompVis/stable-diffusion",

packages/backend/src/constants.ts

@@ -6,7 +6,7 @@ import { Settings } from "./types.js";
 export const DEFAULT_SETTINGS: Settings = {
     maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
     autoDeleteStaleRepos: true,
-    reindexIntervalMs: 1000 * 60,
+    reindexIntervalMs: 1000 * 60 * 60, // 1 hour
     resyncConnectionPollingIntervalMs: 1000,
     reindexRepoPollingIntervalMs: 1000,
     indexConcurrencyMultiple: 3,

packages/backend/src/environment.ts

@@ -18,6 +18,9 @@ export const getEnvBoolean = (env: string | undefined, defaultValue: boolean) =>
 dotenv.config({
 	path: './.env',
 });
+dotenv.config({
+	path: './.env.local',
+});
 
 
 export const SOURCEBOT_LOG_LEVEL = getEnv(process.env.SOURCEBOT_LOG_LEVEL, 'info')!;
@@ -26,3 +29,7 @@ export const SOURCEBOT_INSTALL_ID = getEnv(process.env.SOURCEBOT_INSTALL_ID, 'un
 export const SOURCEBOT_VERSION = getEnv(process.env.SOURCEBOT_VERSION, 'unknown')!;
 export const POSTHOG_PAPIK = getEnv(process.env.POSTHOG_PAPIK);
 export const POSTHOG_HOST = getEnv(process.env.POSTHOG_HOST);
+
+export const FALLBACK_GITHUB_TOKEN = getEnv(process.env.FALLBACK_GITHUB_TOKEN);
+export const FALLBACK_GITLAB_TOKEN = getEnv(process.env.FALLBACK_GITLAB_TOKEN);
+export const FALLBACK_GITEA_TOKEN = getEnv(process.env.FALLBACK_GITEA_TOKEN);

packages/backend/src/gerrit.ts

@@ -2,7 +2,7 @@ import fetch from 'cross-fetch';
 import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
 import { createLogger } from './logger.js';
 import micromatch from "micromatch";
-import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
+import { measure, marshalBool, excludeReposByName, includeReposByName, fetchWithRetry } from './utils.js';
 
 // https://gerrit-review.googlesource.com/Documentation/rest-api.html
 interface GerritProjects {
@@ -30,13 +30,13 @@ interface GerritWebLink {
 const logger = createLogger('Gerrit');
 
 export const getGerritReposFromConfig = async (config: GerritConfig): Promise<GerritProject[]> => {
-
    const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
    const hostname = new URL(config.url).hostname;
 
    let { durationMs, data: projects } = await measure(async () => {
       try {
-         return fetchAllProjects(url)
+         const fetchFn = () => fetchAllProjects(url);
+         return fetchWithRetry(fetchFn, `projects from ${url}`, logger);
       } catch (err) {
          logger.error(`Failed to fetch projects from ${url}`, err);
          return null;

packages/backend/src/git.ts

@@ -99,7 +99,7 @@ export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) =
                 .filter(Boolean)
                 .map(branch => branch.replace('refs/heads/', ''));
 
-            repo.branches = branches.filter(branch => 
+            repo.branches = branches.filter(branch =>
                 branchGlobs.some(glob => new RegExp(glob).test(branch))
             );
         }
@@ -114,7 +114,7 @@ export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) =
                 .filter(Boolean)
                 .map(tag => tag.replace('refs/tags/', ''));
 
-            repo.tags = tags.filter(tag => 
+            repo.tags = tags.filter(tag =>
                 tagGlobs.some(glob => new RegExp(glob).test(tag))
             );
         }

packages/backend/src/gitea.ts

@@ -1,36 +1,47 @@
 import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
 import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
-import { getTokenFromConfig, measure } from './utils.js';
+import { getTokenFromConfig, measure, fetchWithRetry } from './utils.js';
 import fetch from 'cross-fetch';
 import { createLogger } from './logger.js';
 import micromatch from 'micromatch';
 import { PrismaClient } from '@sourcebot/db';
-
+import { FALLBACK_GITEA_TOKEN } from './environment.js';
 const logger = createLogger('Gitea');
 
 export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
-    // TODO: pass in DB here to fetch secret properly
     const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
 
     const api = giteaApi(config.url ?? 'https://gitea.com', {
-        token,
+        token: token ?? FALLBACK_GITEA_TOKEN,
         customFetch: fetch,
     });
 
     let allRepos: GiteaRepository[] = [];
 
     if (config.orgs) {
-        const _repos = await getReposForOrgs(config.orgs, api);
+        const _repos = await fetchWithRetry(
+            () => getReposForOrgs(config.orgs!, api),
+            `orgs ${config.orgs.join(', ')}`,
+            logger
+        );
         allRepos = allRepos.concat(_repos);
     }
 
     if (config.repos) {
-        const _repos = await getRepos(config.repos, api);
+        const _repos = await fetchWithRetry(
+            () => getRepos(config.repos!, api),
+            `repos ${config.repos.join(', ')}`,
+            logger
+        );
         allRepos = allRepos.concat(_repos);
     }
 
     if (config.users) {
-        const _repos = await getReposOwnedByUsers(config.users, api);
+        const _repos = await fetchWithRetry(
+            () => getReposOwnedByUsers(config.users!, api),
+            `users ${config.users.join(', ')}`,
+            logger
+        );
         allRepos = allRepos.concat(_repos);
     }
 
@@ -50,7 +61,11 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
             allRepos = await Promise.all(
                 allRepos.map(async (repo) => {
                     const [owner, name] = repo.full_name!.split('/');
-                    let branches = (await getBranchesForRepo(owner, name, api)).map(branch => branch.name!);
+                    let branches = (await fetchWithRetry(
+                        () => getBranchesForRepo(owner, name, api),
+                        `branches for ${owner}/${name}`,
+                        logger
+                    )).map(branch => branch.name!);
                     branches = micromatch.match(branches, branchGlobs);
 
                     return {
@@ -66,7 +81,11 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
             allRepos = await Promise.all(
                 allRepos.map(async (allRepos) => {
                     const [owner, name] = allRepos.name!.split('/');
-                    let tags = (await getTagsForRepo(owner, name, api)).map(tag => tag.name!);
+                    let tags = (await fetchWithRetry(
+                        () => getTagsForRepo(owner, name, api),
+                        `tags for ${owner}/${name}`,
+                        logger
+                    )).map(tag => tag.name!);
                     tags = micromatch.match(tags, tagGlobs);
 
                     return {

packages/backend/src/github.ts

@@ -1,11 +1,10 @@
 import { Octokit } from "@octokit/rest";
 import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
 import { createLogger } from "./logger.js";
-import { AppContext } from "./types.js";
-import { getTokenFromConfig, measure } from "./utils.js";
+import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
 import micromatch from "micromatch";
 import { PrismaClient } from "@sourcebot/db";
-
+import { FALLBACK_GITHUB_TOKEN } from "./environment.js";
 const logger = createLogger("GitHub");
 
 export type OctokitRepository = {
@@ -33,7 +32,7 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o
     const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
 
     const octokit = new Octokit({
-        auth: token,
+        auth: token ?? FALLBACK_GITHUB_TOKEN,
         ...(config.url ? {
             baseUrl: `${config.url}/api/v3`
         } : {}),
@@ -78,7 +77,7 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o
 
 export const getGitHubRepoFromId = async (id: string, hostURL: string, token?: string) => {
     const octokit = new Octokit({
-        auth: token,
+        auth: token ?? FALLBACK_GITHUB_TOKEN,
         ...(hostURL !== 'https://github.com' ? {
             baseUrl: `${hostURL}/api/v3`
         } : {})
@@ -182,31 +181,34 @@ const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, o
             logger.debug(`Fetching repository info for user ${user}...`);
 
             const { durationMs, data } = await measure(async () => {
-                if (isAuthenticated) {
-                    return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
-                        username: user,
-                        visibility: 'all',
-                        affiliation: 'owner',
-                        per_page: 100,
-                        request: {
-                            signal,
-                        },
-                    });
-                } else {
-                    return octokit.paginate(octokit.repos.listForUser, {
-                        username: user,
-                        per_page: 100,
-                        request: {
-                            signal,
-                        },
-                    });
-                }
+                const fetchFn = async () => {
+                    if (isAuthenticated) {
+                        return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
+                            username: user,
+                            visibility: 'all',
+                            affiliation: 'owner',
+                            per_page: 100,
+                            request: {
+                                signal,
+                            },
+                        });
+                    } else {
+                        return octokit.paginate(octokit.repos.listForUser, {
+                            username: user,
+                            per_page: 100,
+                            request: {
+                                signal,
+                            },
+                        });
+                    }
+                };
+
+                return fetchWithRetry(fetchFn, `user ${user}`, logger);
             });
 
             logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`);
             return data;
         } catch (e) {
-            // @todo: handle rate limiting errors
             logger.error(`Failed to fetch repository info for user ${user}.`, e);
             throw e;
         }
@@ -218,20 +220,23 @@ const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, o
 const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => {
     const repos = (await Promise.all(orgs.map(async (org) => {
         try {
-            logger.debug(`Fetching repository info for org ${org}...`);
+            logger.info(`Fetching repository info for org ${org}...`);
 
-            const { durationMs, data } = await measure(() => octokit.paginate(octokit.repos.listForOrg, {
-                org: org,
-                per_page: 100,
-                request: {
-                    signal
-                }
-            }));
+            const { durationMs, data } = await measure(async () => {
+                const fetchFn = () => octokit.paginate(octokit.repos.listForOrg, {
+                    org: org,
+                    per_page: 100,
+                    request: {
+                        signal
+                    }
+                });
+
+                return fetchWithRetry(fetchFn, `org ${org}`, logger);
+            });
 
-            logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
+            logger.info(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
             return data;
         } catch (e) {
-            // @todo: handle rate limiting errors
             logger.error(`Failed to fetch repository info for org ${org}.`, e);
             throw e;
         }
@@ -243,22 +248,25 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi
 const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => {
     const repos = (await Promise.all(repoList.map(async (repo) => {
         try {
-            logger.debug(`Fetching repository info for ${repo}...`);
-
             const [owner, repoName] = repo.split('/');
-            const { durationMs, data: result } = await measure(() => octokit.repos.get({
-                owner,
-                repo: repoName,
-                request: {
-                    signal
-                }
-            }));
+            logger.info(`Fetching repository info for ${repo}...`);
+
+            const { durationMs, data: result } = await measure(async () => {
+                const fetchFn = () => octokit.repos.get({
+                    owner,
+                    repo: repoName,
+                    request: {
+                        signal
+                    }
+                });
+
+                return fetchWithRetry(fetchFn, repo, logger);
+            });
 
-            logger.debug(`Found info for repository ${repo} in ${durationMs}ms`);
+            logger.info(`Found info for repository ${repo} in ${durationMs}ms`);
 
             return [result.data];
         } catch (e) {
-            // @todo: handle rate limiting errors
             logger.error(`Failed to fetch repository info for ${repo}.`, e);
             throw e;
         }