Connection management

packages/backend/src/connectionManager.ts

@@ -80,54 +80,68 @@ export class ConnectionManager implements IConnectionManager {
         const abortController = new AbortController();
 
         type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
-        const repoData: RepoData[] = await (async () => {
-            switch (config.type) {
-                case 'github': {
-                    const gitHubRepos = await getGitHubReposFromConfig(config, orgId, this.db, abortController.signal);
-                    const hostUrl = config.url ?? 'https://github.com';
-                    const hostname = config.url ? new URL(config.url).hostname : 'github.com';
-
-                    return gitHubRepos.map((repo) => {
-                        const repoName = `${hostname}/${repo.full_name}`;
-                        const cloneUrl = new URL(repo.clone_url!);
-
-                        const record: RepoData = {
-                            external_id: repo.id.toString(),
-                            external_codeHostType: 'github',
-                            external_codeHostUrl: hostUrl,
-                            cloneUrl: cloneUrl.toString(),
-                            name: repoName,
-                            isFork: repo.fork,
-                            isArchived: !!repo.archived,
-                            org: {
-                                connect: {
-                                    id: orgId,
+        const repoData: RepoData[] = (
+            await (async () => {
+                switch (config.type) {
+                    case 'github': {
+                        const gitHubRepos = await getGitHubReposFromConfig(config, orgId, this.db, abortController.signal);
+                        const hostUrl = config.url ?? 'https://github.com';
+                        const hostname = config.url ? new URL(config.url).hostname : 'github.com';
+
+                        return gitHubRepos.map((repo) => {
+                            const repoName = `${hostname}/${repo.full_name}`;
+                            const cloneUrl = new URL(repo.clone_url!);
+
+                            const record: RepoData = {
+                                external_id: repo.id.toString(),
+                                external_codeHostType: 'github',
+                                external_codeHostUrl: hostUrl,
+                                cloneUrl: cloneUrl.toString(),
+                                imageUrl: repo.owner.avatar_url,
+                                name: repoName,
+                                isFork: repo.fork,
+                                isArchived: !!repo.archived,
+                                org: {
+                                    connect: {
+                                        id: orgId,
+                                    },
                                 },
-                            },
-                            connections: {
-                                create: {
-                                    connectionId: job.data.connectionId,
-                                }
-                            },
-                            metadata: {
-                                'zoekt.web-url-type': 'github',
-                                'zoekt.web-url': repo.html_url,
-                                'zoekt.name': repoName,
-                                'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
-                                'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
-                                'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
-                                'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
-                                'zoekt.archived': marshalBool(repo.archived),
-                                'zoekt.fork': marshalBool(repo.fork),
-                                'zoekt.public': marshalBool(repo.private === false)
-                            },
-                        };
-
-                        return record;
-                    })
+                                connections: {
+                                    create: {
+                                        connectionId: job.data.connectionId,
+                                    }
+                                },
+                                metadata: {
+                                    'zoekt.web-url-type': 'github',
+                                    'zoekt.web-url': repo.html_url,
+                                    'zoekt.name': repoName,
+                                    'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
+                                    'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
+                                    'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
+                                    'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
+                                    'zoekt.archived': marshalBool(repo.archived),
+                                    'zoekt.fork': marshalBool(repo.fork),
+                                    'zoekt.public': marshalBool(repo.private === false)
+                                },
+                            };
+
+                            return record;
+                        })
+                    }
+                    case 'gitlab': {
+                        // @todo
+                        return [];
+                    }
                 }
-            }
-        })();
+            })()
+        )
+        // Filter out any duplicates by external_id and external_codeHostUrl.
+        .filter((repo, index, self) => {
+            return index === self.findIndex(r =>
+                r.external_id === repo.external_id &&
+                r.external_codeHostUrl === repo.external_codeHostUrl
+            );
+        })
 
         // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection,
         // and then recreate them when we upsert the repos. For example, if a repo is no-longer

packages/backend/src/github.ts

@@ -24,6 +24,9 @@ export type OctokitRepository = {
     topics?: string[],
     // @note: this is expressed in kilobytes.
     size?: number,
+    owner: {
+        avatar_url: string,
+    }
 }
 
 export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {

packages/db/prisma/migrations/20250124230248_add_name_to_connection/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `name` to the `Connection` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "Connection" ADD COLUMN     "name" TEXT NOT NULL;

packages/db/prisma/migrations/20250124231224_add_connection_type/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `connectionType` to the `Connection` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "Connection" ADD COLUMN     "connectionType" TEXT NOT NULL;

packages/db/prisma/migrations/20250203215003_add_repo_image/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Repo" ADD COLUMN     "imageUrl" TEXT;

packages/db/prisma/schema.prisma

@@ -27,17 +27,17 @@ enum ConnectionSyncStatus {
 }
 
 model Repo {
-  id         Int       @id @default(autoincrement())
-  name       String
-  createdAt  DateTime  @default(now())
-  updatedAt  DateTime  @updatedAt
-  indexedAt  DateTime?
-  isFork     Boolean
-  isArchived Boolean
-  metadata   Json
-  cloneUrl   String
-  connections RepoToConnection[]
-
+  id                 Int                @id @default(autoincrement())
+  name               String
+  createdAt          DateTime           @default(now())
+  updatedAt          DateTime           @updatedAt
+  indexedAt          DateTime?
+  isFork             Boolean
+  isArchived         Boolean
+  metadata           Json
+  cloneUrl           String
+  connections        RepoToConnection[]
+  imageUrl           String?
   repoIndexingStatus RepoIndexingStatus @default(NEW)
 
   // The id of the repo in the external service
@@ -54,15 +54,18 @@ model Repo {
 }
 
 model Connection {
-  id        Int       @id @default(autoincrement())
-  config    Json
-  createdAt DateTime  @default(now())
-  updatedAt DateTime  @updatedAt
-  syncedAt  DateTime?
-  repos RepoToConnection[]
-
+  id         Int                  @id @default(autoincrement())
+  name       String
+  config     Json
+  createdAt  DateTime             @default(now())
+  updatedAt  DateTime             @updatedAt
+  syncedAt   DateTime?
+  repos      RepoToConnection[]
   syncStatus ConnectionSyncStatus @default(SYNC_NEEDED)
 
+  // The type of connection (e.g., github, gitlab, etc.)
+  connectionType String
+
   // The organization that owns this connection
   org   Org @relation(fields: [orgId], references: [id], onDelete: Cascade)
   orgId Int
@@ -71,10 +74,10 @@ model Connection {
 model RepoToConnection {
   addedAt DateTime @default(now())
 
-  connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade)
+  connection   Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade)
   connectionId Int
 
-  repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade)
+  repo   Repo @relation(fields: [repoId], references: [id], onDelete: Cascade)
   repoId Int
 
   @@id([connectionId, repoId])
@@ -113,12 +116,12 @@ model UserToOrg {
 }
 
 model Secret {
-  orgId               Int
-  key                 String
-  encryptedValue      String
-  iv                  String
+  orgId          Int
+  key            String
+  encryptedValue String
+  iv             String
 
-  createdAt           DateTime @default(now())
+  createdAt DateTime @default(now())
 
   org Org @relation(fields: [orgId], references: [id], onDelete: Cascade)
 

packages/schemas/src/v3/connection.schema.ts

@@ -214,6 +214,144 @@ const schema = {
         "type"
       ],
       "additionalProperties": false
+    },
+    {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "type": "object",
+      "title": "GitLabConnectionConfig",
+      "properties": {
+        "type": {
+          "const": "gitlab",
+          "description": "GitLab Configuration"
+        },
+        "token": {
+          "$ref": "#/oneOf/0/properties/token",
+          "description": "An authentication token.",
+          "examples": [
+            "secret-token",
+            {
+              "env": "ENV_VAR_CONTAINING_TOKEN"
+            }
+          ]
+        },
+        "url": {
+          "type": "string",
+          "format": "url",
+          "default": "https://gitlab.com",
+          "description": "The URL of the GitLab host. Defaults to https://gitlab.com",
+          "examples": [
+            "https://gitlab.com",
+            "https://gitlab.example.com"
+          ],
+          "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
+        },
+        "all": {
+          "type": "boolean",
+          "default": false,
+          "description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ."
+        },
+        "users": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property."
+        },
+        "groups": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": [
+            [
+              "my-group"
+            ],
+            [
+              "my-group/sub-group-a",
+              "my-group/sub-group-b"
+            ]
+          ],
+          "description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)."
+        },
+        "projects": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": [
+            [
+              "my-group/my-project"
+            ],
+            [
+              "my-group/my-sub-group/my-project"
+            ]
+          ],
+          "description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/"
+        },
+        "topics": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "minItems": 1,
+          "description": "List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.",
+          "examples": [
+            [
+              "docs",
+              "core"
+            ]
+          ]
+        },
+        "exclude": {
+          "type": "object",
+          "properties": {
+            "forks": {
+              "type": "boolean",
+              "default": false,
+              "description": "Exclude forked projects from syncing."
+            },
+            "archived": {
+              "type": "boolean",
+              "default": false,
+              "description": "Exclude archived projects from syncing."
+            },
+            "projects": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "default": [],
+              "examples": [
+                [
+                  "my-group/my-project"
+                ]
+              ],
+              "description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/"
+            },
+            "topics": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.",
+              "examples": [
+                [
+                  "tests",
+                  "ci"
+                ]
+              ]
+            }
+          },
+          "additionalProperties": false
+        },
+        "revisions": {
+          "$ref": "#/oneOf/0/properties/revisions"
+        }
+      },
+      "required": [
+        "type"
+      ],
+      "additionalProperties": false
     }
   ]
 } as const;