add grafana alloy config and setup (#210)

* add grafana alloy config and setup

* add basic repo prom metrics

* nits in dockerfile

Author: msukkari

Dockerfile

@@ -111,9 +111,6 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
 
 ENV STRIPE_PUBLISHABLE_KEY=""
 
-# Configure dependencies
-RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib openssl util-linux 
-
 # Configure zoekt
 COPY vendor/zoekt/install-ctags-alpine.sh .
 RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
@@ -144,6 +141,19 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 
+# Configure dependencies
+RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib openssl util-linux unzip
+
+# Install grafana alloy. libc6-compat is required because alloy dynamically links against glibc which doesn't exist in alpine by default
+# @nochekin: figure out how to handle this for self hosted case (especially the config)
+RUN apk add --no-cache libc6-compat 
+RUN wget https://github.com/grafana/alloy/releases/download/v1.7.0/alloy-linux-amd64.zip \
+    && unzip alloy-linux-amd64.zip \
+    && mv alloy-linux-amd64 /usr/local/bin/alloy \
+    && chmod +x /usr/local/bin/alloy \
+    && rm alloy-linux-amd64.zip
+COPY grafana.alloy .
+
 # Configure the database
 RUN mkdir -p /run/postgresql && \
     chown -R postgres:postgres /run/postgresql && \
@@ -157,7 +167,6 @@ RUN chmod +x ./entrypoint.sh
 
 COPY default-config.json .
 
-
 EXPOSE 3000
 ENV PORT=3000
 ENV HOSTNAME="0.0.0.0"

grafana.alloy

@@ -0,0 +1,30 @@
+prometheus.scrape "local_app" {
+  targets = [
+    {
+      __address__ = "localhost:6070",
+    },
+    {
+      __address__ = "localhost:3060",
+    },
+  ]
+
+  metrics_path = "/metrics"
+  scrape_interval = "10s"
+
+  job_name = sys.env("GRAFANA_ENVIRONMENT")
+  
+  forward_to = [
+    prometheus.remote_write.grafana_cloud.receiver,
+  ]
+}
+
+prometheus.remote_write "grafana_cloud" {
+   endpoint {
+      url  = sys.env("GRAFANA_ENDPOINT")
+  
+      basic_auth {
+        username = sys.env("GRAFANA_USERNAME")
+        password = sys.env("GRAFANA_PASSWORD")
+      }
+   }
+}

packages/backend/package.json

@@ -23,22 +23,25 @@
     "dependencies": {
         "@gitbeaker/rest": "^40.5.1",
         "@octokit/rest": "^21.0.2",
+        "@sourcebot/crypto": "^0.1.0",
+        "@sourcebot/db": "^0.1.0",
+        "@sourcebot/error": "^0.1.0",
+        "@sourcebot/schemas": "^0.1.0",
+        "@types/express": "^5.0.0",
         "argparse": "^2.0.1",
+        "bullmq": "^5.34.10",
         "cross-fetch": "^4.0.0",
         "dotenv": "^16.4.5",
+        "express": "^4.21.2",
         "gitea-js": "^1.22.0",
         "glob": "^11.0.0",
+        "ioredis": "^5.4.2",
         "lowdb": "^7.0.1",
         "micromatch": "^4.0.8",
         "posthog-node": "^4.2.1",
-        "@sourcebot/crypto": "^0.1.0",
-        "@sourcebot/db": "^0.1.0",
-        "@sourcebot/schemas": "^0.1.0",
-        "@sourcebot/error": "^0.1.0",
+        "prom-client": "^15.1.3",
         "simple-git": "^3.27.0",
         "strip-json-comments": "^5.0.1",
-        "winston": "^3.15.0",
-        "bullmq": "^5.34.10",
-        "ioredis": "^5.4.2"
+        "winston": "^3.15.0"
     }
 }

packages/backend/src/main.ts

@@ -6,6 +6,7 @@ import { Redis } from 'ioredis';
 import { ConnectionManager } from './connectionManager.js';
 import { RepoManager } from './repoManager.js';
 import { INDEX_CONCURRENCY_MULTIPLE, REDIS_URL } from './environment.js';
+import { PromClient } from './promClient.js';
 
 const logger = createLogger('main');
 
@@ -26,9 +27,11 @@ export const main = async (db: PrismaClient, context: AppContext) => {
         settings.indexConcurrencyMultiple = parseInt(INDEX_CONCURRENCY_MULTIPLE);
     }
 
+    const promClient = new PromClient();
+
     const connectionManager = new ConnectionManager(db, settings, redis);
     connectionManager.registerPollingCallback();
 
-    const repoManager = new RepoManager(db, settings, redis, context);
+    const repoManager = new RepoManager(db, settings, redis, promClient, context);
     await repoManager.blockingPollLoop();
 }

packages/backend/src/promClient.ts

@@ -0,0 +1,72 @@
+import express, { Request, Response } from 'express';
+import client, { Registry, Counter, Gauge, Histogram } from 'prom-client';
+
+export class PromClient {
+    private registry: Registry;
+    private app: express.Application;
+    public activeRepoIndexingJobs: Gauge<string>;
+    public repoIndexingDuration: Histogram<string>;
+    public repoIndexingErrors: Counter<string>;
+    public repoIndexingFails: Counter<string>;
+    public repoIndexingSuccesses: Counter<string>;
+    public readonly PORT = 3060;
+
+    constructor() {
+        this.registry = new Registry();
+
+        this.activeRepoIndexingJobs = new Gauge({
+            name: 'active_repo_indexing_jobs',
+            help: 'The number of repo indexing jobs in progress',
+            labelNames: ['repo'],
+        });
+        this.registry.registerMetric(this.activeRepoIndexingJobs);
+
+        this.repoIndexingDuration = new Histogram({
+            name: 'repo_indexing_duration',
+            help: 'The duration of repo indexing jobs', 
+            labelNames: ['repo'],
+        });
+        this.registry.registerMetric(this.repoIndexingDuration);
+
+        this.repoIndexingErrors = new Counter({
+            name: 'repo_indexing_errors',
+            help: 'The number of repo indexing errors',
+            labelNames: ['repo'],
+        });
+        this.registry.registerMetric(this.repoIndexingErrors);
+
+        this.repoIndexingFails = new Counter({
+            name: 'repo_indexing_fails',
+            help: 'The number of repo indexing fails',
+            labelNames: ['repo'],
+        });
+        this.registry.registerMetric(this.repoIndexingFails);
+
+        this.repoIndexingSuccesses = new Counter({
+            name: 'repo_indexing_successes',
+            help: 'The number of repo indexing successes',
+            labelNames: ['repo'],
+        });
+        this.registry.registerMetric(this.repoIndexingSuccesses);
+
+        client.collectDefaultMetrics({
+            register: this.registry,
+        });
+
+        this.app = express();
+        this.app.get('/metrics', async (req: Request, res: Response) => {
+            res.set('Content-Type', this.registry.contentType);
+
+            const metrics = await this.registry.metrics();
+            res.end(metrics);
+        });
+
+        this.app.listen(this.PORT, () => {
+            console.log(`Prometheus metrics server is running on port ${this.PORT}`);
+        });
+    }
+
+    getRegistry(): Registry {
+        return this.registry;
+    }
+}

packages/backend/src/repoManager.ts

@@ -4,13 +4,12 @@ import { createLogger } from "./logger.js";
 import { Connection, PrismaClient, Repo, RepoToConnection, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db";
 import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
 import { AppContext, Settings } from "./types.js";
-import { captureEvent } from "./posthog.js";
 import { getRepoPath, getTokenFromConfig, measure, getShardPrefix } from "./utils.js";
 import { cloneRepository, fetchRepository } from "./git.js";
 import { existsSync, rmSync, readdirSync } from 'fs';
 import { indexGitRepository } from "./zoekt.js";
 import os from 'os';
-import { BackendException } from "@sourcebot/error";
+import { PromClient } from './promClient.js';
 
 interface IRepoManager {
     blockingPollLoop: () => void;
@@ -34,6 +33,7 @@ export class RepoManager implements IRepoManager {
         private db: PrismaClient,
         private settings: Settings,
         redis: Redis,
+        private promClient: PromClient,
         private ctx: AppContext,
     ) {
         this.queue = new Queue<JobPayload>(QUEUE_NAME, {
@@ -280,6 +280,7 @@ export class RepoManager implements IRepoManager {
                 repoIndexingStatus: RepoIndexingStatus.INDEXING,
             }
         });
+        this.promClient.activeRepoIndexingJobs.inc();
 
         let indexDuration_s: number | undefined;
         let fetchDuration_s: number | undefined;
@@ -295,6 +296,7 @@ export class RepoManager implements IRepoManager {
                 break;
             } catch (error) {
                 attempts++;
+                this.promClient.repoIndexingErrors.inc();
                 if (attempts === maxAttempts) {
                     this.logger.error(`Failed to sync repository ${repo.id} after ${maxAttempts} attempts. Error: ${error}`);
                     throw error;
@@ -313,7 +315,9 @@ export class RepoManager implements IRepoManager {
 
     private async onIndexJobCompleted(job: Job<JobPayload>) {
         this.logger.info(`Repo index job ${job.id} completed`);
-    
+        this.promClient.activeRepoIndexingJobs.dec();
+        this.promClient.repoIndexingSuccesses.inc();
+        
         await this.db.repo.update({
             where: {
                 id: job.data.repo.id,
@@ -328,6 +332,9 @@ export class RepoManager implements IRepoManager {
     private async onIndexJobFailed(job: Job<JobPayload> | undefined, err: unknown) {
         this.logger.info(`Repo index job failed (id: ${job?.id ?? 'unknown'}) with error: ${err}`);
         if (job) {
+            this.promClient.activeRepoIndexingJobs.dec();
+            this.promClient.repoIndexingFails.inc();
+
             await this.db.repo.update({
                 where: {
                     id: job.data.repo.id,

supervisord.conf

@@ -37,4 +37,13 @@ autorestart=true
 startretries=3
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:alloy]
+command=alloy run grafana.alloy
+autostart=true
+autorestart=true
+startretries=3
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
 redirect_stderr=true