Support for github enterprise server(self hosted github) (#646)

* ui: add optional Custom Domain input in GH integration

* feat(github): add support for custom GitHub domains in API service

* feat(github): validate custom domain input with regex and provide user feedback

* feat(github): update GithubApiService to handle optional domain input and set default API URL

* feat(github): refactor GitHub domain handling to use centralized utility function

* fix(github): update GITHUB_API_URL to use REST API endpoint and add error handling for custom domain retrieval

* cleanup and test config

* Error Handling

Author: thesujai

backend/analytics_server/mhq/exapi/github.py

@@ -22,12 +22,18 @@ class GithubRateLimitExceeded(Exception):
 
 
 class GithubApiService:
-    def __init__(self, access_token: str):
+    def __init__(self, access_token: str, domain: Optional[str]):
         self._token = access_token
-        self._g = Github(self._token, per_page=PAGE_SIZE)
-        self.base_url = "https://api.github.com"
+        self.base_url = self._get_api_url(domain)
+        self._g = Github(self._token, base_url=self.base_url, per_page=PAGE_SIZE)
         self.headers = {"Authorization": f"Bearer {self._token}"}
 
+    def _get_api_url(self, domain: str) -> str:
+        if not domain:
+            return "https://api.github.com"
+        else:
+            return f"{domain}/api/v3"
+
     @contextlib.contextmanager
     def temp_config(self, per_page: int = 30):
         self._g.per_page = per_page

backend/analytics_server/mhq/service/code/sync/etl_github_handler.py

@@ -3,6 +3,7 @@
 from typing import List, Dict, Optional, Tuple, Set
 
 import pytz
+from mhq.utils.github import get_custom_github_domain
 from github.PaginatedList import PaginatedList as GithubPaginatedList
 from github.PullRequest import PullRequest as GithubPullRequest
 from github.PullRequestReview import PullRequestReview as GithubPullRequestReview
@@ -382,7 +383,7 @@ def _get_access_token():
 
     return GithubETLHandler(
         org_id,
-        GithubApiService(_get_access_token()),
+        GithubApiService(_get_access_token(), get_custom_github_domain(org_id)),
         CodeRepoService(),
         CodeETLAnalyticsService(),
         get_revert_prs_github_sync_handler(),

backend/analytics_server/mhq/service/external_integrations_service.py

@@ -26,29 +26,29 @@ def __init__(
         self.custom_domain = custom_domain
 
     def get_github_organizations(self):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             orgs: [GithubOrganization] = github_api_service.get_org_list()
         except GithubException as e:
             raise e
         return orgs
 
     def get_github_org_repos(self, org_login: str, page_size: int, page: int):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             return github_api_service.get_repos_raw(org_login, page_size, page)
         except GithubException as e:
             raise e
 
     def get_github_personal_repos(self, page_size: int, page: int):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             return github_api_service.get_user_repos_raw(page_size, page)
         except GithubException as e:
             raise e
 
     def get_repo_workflows(self, gh_org_name: str, gh_org_repo_name: str):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             workflows = github_api_service.get_repo_workflows(
                 gh_org_name, gh_org_repo_name

backend/analytics_server/mhq/service/workflows/sync/etl_github_actions_handler.py

@@ -4,6 +4,7 @@
 
 import pytz
 
+from mhq.utils.github import get_custom_github_domain
 from mhq.exapi.github import GithubApiService
 from mhq.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler
 from mhq.store.models import UserIdentityProvider
@@ -181,5 +182,7 @@ def _get_access_token():
         return access_token
 
     return GithubActionsETLHandler(
-        org_id, GithubApiService(_get_access_token()), WorkflowRepoService()
+        org_id,
+        GithubApiService(_get_access_token(), get_custom_github_domain(org_id)),
+        WorkflowRepoService(),
     )

backend/analytics_server/mhq/utils/github.py

@@ -1,9 +1,12 @@
 from queue import Queue
 from threading import Thread
+from typing import Optional
 
 from github import Organization
 
 from mhq.utils.log import LOG
+from mhq.store.repos.core import CoreRepoService
+from mhq.store.models import UserIdentityProvider
 
 
 def github_org_data_multi_thread_worker(orgs: [Organization]) -> dict:
@@ -48,3 +51,25 @@ def run(self):
     for worker in workers:
         r.update(worker.results)
     return r
+
+
+def get_custom_github_domain(org_id: str) -> Optional[str]:
+    DEFAULT_DOMAIN = "https://api.github.com"
+    core_repo_service = CoreRepoService()
+    integrations = core_repo_service.get_org_integrations_for_names(
+        org_id, [UserIdentityProvider.GITHUB.value]
+    )
+
+    github_domain = (
+        integrations[0].provider_meta.get("custom_domain")
+        if integrations[0].provider_meta
+        else None
+    )
+
+    if not github_domain:
+        LOG.warn(
+            f"Custom domain not found for intergration for org {org_id} and provider {UserIdentityProvider.GITHUB.value}"
+        )
+        return DEFAULT_DOMAIN
+
+    return github_domain

backend/analytics_server/tests/exapi/__init__.py

@@ -0,0 +1,40 @@
+import unittest
+from unittest.mock import patch
+
+from mhq.exapi.github import GithubApiService, PAGE_SIZE
+
+
+class DummyGithub:
+    def __init__(self, token, base_url=None, per_page=None):
+        self.token = token
+        self.base_url = base_url
+        self.per_page = per_page
+
+
+class TestGithubApiService(unittest.TestCase):
+
+    @patch("mhq.exapi.github.Github", new=DummyGithub)
+    def test_default_domain_sets_standard_api_url(self):
+        token = "deadpool"
+        service = GithubApiService(access_token=token, domain=None)
+        self.assertEqual(service.base_url, "https://api.github.com")
+        self.assertIsInstance(service._g, DummyGithub)
+        self.assertEqual(service._g.token, token)
+        self.assertEqual(service._g.base_url, "https://api.github.com")
+        self.assertEqual(service._g.per_page, PAGE_SIZE)
+
+    @patch("mhq.exapi.github.Github", new=DummyGithub)
+    def test_empty_string_domain_uses_default_url(self):
+        token = "deadpool"
+        service = GithubApiService(access_token=token, domain="")
+        self.assertEqual(service.base_url, "https://api.github.com")
+        self.assertEqual(service._g.base_url, "https://api.github.com")
+
+    @patch("mhq.exapi.github.Github", new=DummyGithub)
+    def test_custom_domain_appends_api_v3(self):
+        token = "deadpool"
+        custom_domain = "https://github.sujai.com"
+        service = GithubApiService(access_token=token, domain=custom_domain)
+        expected = f"{custom_domain}/api/v3"
+        self.assertEqual(service.base_url, expected)
+        self.assertEqual(service._g.base_url, expected)

backend/analytics_server/tests/exapi/test_github.py

@@ -1,6 +1,7 @@
 module.exports = {
   preset: 'ts-jest/presets/js-with-babel', // Use the TypeScript preset with Babel
   testEnvironment: 'jsdom', // Use jsdom as the test environment (for browser-like behavior)
+  setupFiles: ['<rootDir>/jest.setup.js'],
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
   testMatch: [
     '**/__tests__/**/*.test.(ts|tsx|js|jsx)',

web-server/jest.config.js

@@ -0,0 +1,3 @@
+const { TextEncoder, TextDecoder } = require('util');
+global.TextEncoder = TextEncoder;
+global.TextDecoder = TextDecoder;

web-server/jest.setup.js

@@ -0,0 +1,84 @@
+jest.mock('@/utils/db', () => ({
+    db: jest.fn(),
+}));
+
+import { db } from '@/utils/db';
+import * as githubUtils from '../utils';
+import { DEFAULT_GH_URL } from '@/constants/urls';
+
+describe('GitHub URL utilities', () => {
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  describe('getGitHubCustomDomain', () => {
+    it('returns custom_domain when present', async () => {
+      const mockMeta = [{ custom_domain: 'custom.sujai.com' }];
+      (db as jest.Mock).mockReturnValue({
+        where: jest.fn().mockReturnThis(),
+        then: jest.fn().mockResolvedValue(mockMeta)
+      });
+
+      const domain = await githubUtils.getGitHubCustomDomain();
+      expect(domain).toBe('custom.sujai.com');
+    });
+
+    it('returns null when no provider_meta found', async () => {
+      (db as jest.Mock).mockReturnValue({
+        where: jest.fn().mockReturnThis(),
+        then: jest.fn().mockResolvedValue([])
+      });
+
+      const domain = await githubUtils.getGitHubCustomDomain();
+      expect(domain).toBeNull();
+    });
+
+    it('returns null on db error and logs error', async () => {
+      const consoleSpy = jest.spyOn(console, 'error').mockImplementation();
+      (db as jest.Mock).mockImplementation(() => {
+        throw new Error('DB failure');
+      });
+
+      const domain = await githubUtils.getGitHubCustomDomain();
+      expect(domain).toBeNull();
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Error occured while getting custom domain from database:',
+        expect.any(Error)
+      );
+    });
+  });
+
+  describe('getGitHubRestApiUrl', () => {
+    it('uses default URL when no custom domain', async () => {
+      jest.spyOn(githubUtils, 'getGitHubCustomDomain').mockResolvedValue(null);
+      const url = await githubUtils.getGitHubRestApiUrl('path/to/repo');
+      expect(url).toBe(`${DEFAULT_GH_URL}/path/to/repo`);
+    });
+
+    it('uses custom domain when provided', async () => {
+      jest.spyOn(githubUtils, 'getGitHubCustomDomain').mockResolvedValue('git.sujai.com');
+      const url = await githubUtils.getGitHubRestApiUrl('repos/owner/repo');
+      expect(url).toBe('https://git.sujai.com/api/v3/repos/owner/repo');
+    });
+
+    it('normalizes multiple slashes in URL', async () => {
+      jest.spyOn(githubUtils, 'getGitHubCustomDomain').mockResolvedValue('git.sujai.com/');
+      const url = await githubUtils.getGitHubRestApiUrl('/repos//owner//repo');
+      expect(url).toBe('https://git.sujai.com/api/v3/repos/owner/repo');
+    });
+  });
+
+  describe('getGitHubGraphQLUrl', () => {
+    it('uses default GraphQL endpoint when no custom domain', async () => {
+      jest.spyOn(githubUtils, 'getGitHubCustomDomain').mockResolvedValue(null);
+      const url = await githubUtils.getGitHubGraphQLUrl();
+      expect(url).toBe(`${DEFAULT_GH_URL}/graphql`);
+    });
+
+    it('uses custom domain for GraphQL endpoint', async () => {
+      jest.spyOn(githubUtils, 'getGitHubCustomDomain').mockResolvedValue('api.github.local');
+      const url = await githubUtils.getGitHubGraphQLUrl();
+      expect(url).toBe('https://api.github.local/api/graphql');
+    });
+  });
+});

web-server/pages/api/internal/[org_id]/__tests__/github.test.ts

@@ -5,8 +5,7 @@ import { Row } from '@/constants/db';
 import { Integration } from '@/constants/integrations';
 import { BaseRepo } from '@/types/resources';
 import { db } from '@/utils/db';
-
-const GITHUB_API_URL = 'https://api.github.com/graphql';
+import { DEFAULT_GH_URL } from '@/constants/urls';
 
 type GithubRepo = {
   name: string;
@@ -53,7 +52,7 @@ export const searchGithubRepos = async (
 };
 
 const searchRepoWithURL = async (searchString: string) => {
-  const apiUrl = `https://api.github.com/repos/${searchString}`;
+  const apiUrl = await getGitHubRestApiUrl(`repos/${searchString}`);
   const response = await axios.get<GithubRepo>(apiUrl);
   const repo = response.data;
   return [
@@ -104,7 +103,9 @@ export const searchGithubReposWithNames = async (
 
   const queryString = `${searchString} in:name fork:true`;
 
-  const response = await fetch(GITHUB_API_URL, {
+  const githubApiUrl = await getGitHubGraphQLUrl();
+
+  const response = await fetch(githubApiUrl, {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json',
@@ -304,3 +305,33 @@ const replaceURL = async (url: string): Promise<string> => {
 
   return url;
 };
+
+export const getGitHubCustomDomain = async (): Promise<string | null> => {
+  try {
+    const provider_meta = await db('Integration')
+      .where('name', Integration.GITHUB)
+      .then((r: Row<'Integration'>[]) => r.map((item) => item.provider_meta));
+
+    return head(provider_meta || [])?.custom_domain || null;
+  } catch (error) {
+    console.error('Error occured while getting custom domain from database:', error);
+    return null;
+  }
+};
+
+const normalizeSlashes = (url: string) =>
+  url.replace(/(?<!:)\/{2,}/g, '/');
+
+export const getGitHubRestApiUrl = async (path: string) => {
+  const customDomain = await getGitHubCustomDomain();
+  const base = customDomain
+    ? `${customDomain}/api/v3`
+    : DEFAULT_GH_URL;
+  return normalizeSlashes(`${base}/${path}`);
+};
+
+
+export const getGitHubGraphQLUrl = async (): Promise<string> => {
+  const customDomain = await getGitHubCustomDomain();
+  return customDomain ? `${customDomain}/api/graphql` : `${DEFAULT_GH_URL}/graphql`;
+};

web-server/pages/api/internal/[org_id]/utils.ts

@@ -0,0 +1 @@
+export const DEFAULT_GH_URL = 'https://api.github.com';