Add @sourcebot/schemas package

Dockerfile

@@ -10,13 +10,15 @@ RUN go mod download
 COPY vendor/zoekt ./
 RUN CGO_ENABLED=0 GOOS=linux go build -o /cmd/ ./cmd/...
 
-# ------ Build Database ------
-FROM node-alpine AS database-builder
+# ------ Build shared libraries ------
+FROM node-alpine AS shared-libs-builder
 WORKDIR /app
 
 COPY package.json yarn.lock* ./
 COPY ./packages/db ./packages/db
+COPY ./packages/schemas ./packages/schemas
 RUN yarn workspace @sourcebot/db install --frozen-lockfile
+RUN yarn workspace @sourcebot/schemas install --frozen-lockfile
 
 # ------ Build Web ------
 FROM node-alpine AS web-builder
@@ -25,8 +27,9 @@ WORKDIR /app
 
 COPY package.json yarn.lock* ./
 COPY ./packages/web ./packages/web
-COPY --from=database-builder /app/node_modules ./node_modules
-COPY --from=database-builder /app/packages/db ./packages/db
+COPY --from=shared-libs-builder /app/node_modules ./node_modules
+COPY --from=shared-libs-builder /app/packages/db ./packages/db
+COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 
 # Fixes arm64 timeouts
 RUN yarn config set registry https://registry.npmjs.org/
@@ -54,8 +57,9 @@ WORKDIR /app
 COPY package.json yarn.lock* ./
 COPY ./schemas ./schemas
 COPY ./packages/backend ./packages/backend
-COPY --from=database-builder /app/node_modules ./node_modules
-COPY --from=database-builder /app/packages/db ./packages/db
+COPY --from=shared-libs-builder /app/node_modules ./node_modules
+COPY --from=shared-libs-builder /app/packages/db ./packages/db
+COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 RUN yarn workspace @sourcebot/backend install --frozen-lockfile
 RUN yarn workspace @sourcebot/backend build
 
@@ -114,21 +118,22 @@ COPY --from=zoekt-builder \
 /cmd/zoekt-index \
 /usr/local/bin/
 
-# Configure the webapp
+# Copy all of the things
 COPY --from=web-builder /app/packages/web/public ./packages/web/public
 COPY --from=web-builder /app/packages/web/.next/standalone ./
 COPY --from=web-builder /app/packages/web/.next/static ./packages/web/.next/static
 
-# Configure the backend
 COPY --from=backend-builder /app/node_modules ./node_modules
 COPY --from=backend-builder /app/packages/backend ./packages/backend
 
+COPY --from=shared-libs-builder /app/node_modules ./node_modules
+COPY --from=shared-libs-builder /app/packages/db ./packages/db
+COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
+
 # Configure the database
 RUN mkdir -p /run/postgresql && \
     chown -R postgres:postgres /run/postgresql && \
     chmod 775 /run/postgresql
-COPY --from=database-builder /app/node_modules ./node_modules
-COPY --from=database-builder /app/packages/db ./packages/db
 
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 COPY prefix-output.sh ./prefix-output.sh

Makefile

@@ -22,6 +22,8 @@ clean:
 		packages/backend/node_modules \
 		packages/db/node_modules \
 		packages/db/dist \
+		packages/schemas/node_modules \
+		packages/schemas/dist \
 		.sourcebot
 
 .PHONY: bin

packages/backend/package.json

@@ -5,10 +5,9 @@
     "main": "index.js",
     "type": "module",
     "scripts": {
-        "dev:watch": "yarn generate:types && tsc-watch --preserveWatchOutput --onSuccess \"yarn dev --configPath ../../config.json --cacheDir ../../.sourcebot\"",
+        "dev:watch": "tsc-watch --preserveWatchOutput --onSuccess \"yarn dev --configPath ../../config.json --cacheDir ../../.sourcebot\"",
         "dev": "export PATH=\"$PWD/../../bin:$PATH\" && export CTAGS_COMMAND=ctags && node ./dist/index.js",
-        "build": "yarn generate:types && tsc",
-        "generate:types": "tsx tools/generateTypes.ts",
+        "build": "tsc",
         "test": "vitest --config ./vitest.config.ts"
     },
     "devDependencies": {
@@ -33,6 +32,7 @@
         "micromatch": "^4.0.8",
         "posthog-node": "^4.2.1",
         "@sourcebot/db": "^0.1.0",
+        "@sourcebot/schemas": "^0.1.0",
         "simple-git": "^3.27.0",
         "strip-json-comments": "^5.0.1",
         "winston": "^3.15.0",

packages/backend/src/config.ts

@@ -3,7 +3,7 @@ import { readFile } from 'fs/promises';
 import stripJsonComments from 'strip-json-comments';
 import { getGitHubReposFromConfig } from "./github.js";
 import { getGitLabReposFromConfig, GITLAB_CLOUD_HOSTNAME } from "./gitlab.js";
-import { SourcebotConfigurationSchema } from "./schemas/v2.js";
+import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type";
 import { AppContext } from "./types.js";
 import { getTokenFromConfig, isRemotePath, marshalBool } from "./utils.js";
 

packages/backend/src/gerrit.ts

@@ -1,5 +1,5 @@
 import fetch from 'cross-fetch';
-import { GerritConfig } from './schemas/v2.js';
+import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
 import { AppContext, GitRepository } from './types.js';
 import { createLogger } from './logger.js';
 import path from 'path';

packages/backend/src/git.ts

@@ -1,7 +1,7 @@
 import { GitRepository, AppContext } from './types.js';
 import { simpleGit, SimpleGitProgressEvent } from 'simple-git';
 import { createLogger } from './logger.js';
-import { GitConfig } from './schemas/v2.js';
+import { GitConfig } from "@sourcebot/schemas/v2/index.type"
 import path from 'path';
 
 const logger = createLogger('git');

packages/backend/src/gitea.ts

@@ -1,5 +1,5 @@
 import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
-import { GiteaConfig } from './schemas/v2.js';
+import { GiteaConfig } from "@sourcebot/schemas/v2/index.type"
 import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js';
 import { AppContext, GitRepository } from './types.js';
 import fetch from 'cross-fetch';

packages/backend/src/github.ts

@@ -1,5 +1,5 @@
 import { Octokit } from "@octokit/rest";
-import { GitHubConfig } from "./schemas/v2.js";
+import { GitHubConfig } from "@sourcebot/schemas/v2/index.type"
 import { createLogger } from "./logger.js";
 import { AppContext } from "./types.js";
 import { getTokenFromConfig, measure } from "./utils.js";

packages/backend/src/gitlab.ts

@@ -1,7 +1,7 @@
 import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
 import micromatch from "micromatch";
 import { createLogger } from "./logger.js";
-import { GitLabConfig } from "./schemas/v2.js";
+import { GitLabConfig } from "@sourcebot/schemas/v2/index.type"
 import { AppContext } from "./types.js";
 import { getTokenFromConfig, measure } from "./utils.js";
 

packages/backend/src/local.ts

@@ -1,6 +1,6 @@
 import { existsSync, FSWatcher, statSync, watch } from "fs";
 import { createLogger } from "./logger.js";
-import { LocalConfig } from "./schemas/v2.js";
+import { LocalConfig } from "@sourcebot/schemas/v2/index.type"
 import { AppContext, LocalRepository } from "./types.js";
 import { resolvePathRelativeToConfig } from "./utils.js";
 import path from "path";

packages/backend/src/main.ts

@@ -12,7 +12,7 @@ import { Queue, Worker, Job } from 'bullmq';
 import { Redis } from 'ioredis';
 import * as os from 'os';
 import { SOURCEBOT_TENANT_MODE } from './environment.js';
-import { SourcebotConfigurationSchema } from './schemas/v2.js';
+import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type"
 
 const logger = createLogger('main');
 

packages/schemas/package.json

@@ -0,0 +1,21 @@
+{
+    "name": "@sourcebot/schemas",
+    "version": "0.1.0",
+    "private": true,
+    "scripts": {
+        "build": "yarn generate && tsc",
+        "generate": "tsx tools/generate.ts",
+        "postinstall": "yarn build"
+    },
+    "devDependencies": {
+        "@apidevtools/json-schema-ref-parser": "^11.7.3",
+        "glob": "^11.0.1",
+        "json-schema-to-typescript": "^15.0.4",
+        "tsx": "^4.19.2",
+        "typescript": "^5.7.3"
+    },
+    "exports": {
+        "./v2/*": "./dist/v2/*.js",
+        "./v3/*": "./dist/v3/*.js"
+    }
+}

packages/schemas/src/v1/index.schema.ts

@@ -0,0 +1,137 @@
+// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
+const schema = {
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "definitions": {
+    "RepoNameRegexIncludeFilter": {
+      "type": "string",
+      "description": "Only clone repos whose name matches the given regexp.",
+      "format": "regexp",
+      "default": "^(foo|bar)$"
+    },
+    "RepoNameRegexExcludeFilter": {
+      "type": "string",
+      "description": "Don't mirror repos whose names match this regexp.",
+      "format": "regexp",
+      "default": "^(fizz|buzz)$"
+    },
+    "ZoektConfig": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/GitHubConfig"
+        },
+        {
+          "$ref": "#/definitions/GitLabConfig"
+        }
+      ]
+    },
+    "GitHubConfig": {
+      "type": "object",
+      "properties": {
+        "Type": {
+          "const": "github"
+        },
+        "GitHubUrl": {
+          "type": "string",
+          "description": "GitHub Enterprise url. If not set github.com will be used as the host."
+        },
+        "GitHubUser": {
+          "type": "string",
+          "description": "The GitHub user to mirror"
+        },
+        "GitHubOrg": {
+          "type": "string",
+          "description": "The GitHub organization to mirror"
+        },
+        "Name": {
+          "$ref": "#/definitions/RepoNameRegexIncludeFilter"
+        },
+        "Exclude": {
+          "$ref": "#/definitions/RepoNameRegexExcludeFilter"
+        },
+        "CredentialPath": {
+          "type": "string",
+          "description": "Path to a file containing a GitHub access token.",
+          "default": "~/.github-token"
+        },
+        "Topics": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Only mirror repos that have one of the given topics"
+        },
+        "ExcludeTopics": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Don't mirror repos that have one of the given topics"
+        },
+        "NoArchived": {
+          "type": "boolean",
+          "description": "Mirror repos that are _not_ archived",
+          "default": false
+        },
+        "IncludeForks": {
+          "type": "boolean",
+          "description": "Also mirror forks",
+          "default": false
+        }
+      },
+      "required": [
+        "Type"
+      ],
+      "additionalProperties": false
+    },
+    "GitLabConfig": {
+      "type": "object",
+      "properties": {
+        "Type": {
+          "const": "gitlab"
+        },
+        "GitLabURL": {
+          "type": "string",
+          "description": "The GitLab API url.",
+          "default": "https://gitlab.com/api/v4/"
+        },
+        "Name": {
+          "$ref": "#/definitions/RepoNameRegexIncludeFilter"
+        },
+        "Exclude": {
+          "$ref": "#/definitions/RepoNameRegexExcludeFilter"
+        },
+        "OnlyPublic": {
+          "type": "boolean",
+          "description": "Only mirror public repos",
+          "default": false
+        },
+        "CredentialPath": {
+          "type": "string",
+          "description": "Path to a file containing a GitLab access token.",
+          "default": "~/.gitlab-token"
+        }
+      },
+      "required": [
+        "Type"
+      ],
+      "additionalProperties": false
+    }
+  },
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "Configs": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/ZoektConfig"
+      }
+    }
+  },
+  "required": [
+    "Configs"
+  ],
+  "additionalProperties": false
+} as const;
+export { schema as indexSchema };