Set up initial metrics reporter. (airbytehq#10408)

Part 1 of https://docs.google.com/document/d/11pEUsHyKUhh4CtV3aReau3SUG-ncEvy6ROJRVln6YB4/edit?usp=sharing.

This is the initial set up of the reporter. Will add actual metrics in the follow up PR.

Since the reporter is primarily for Cloud use and limited to DD to begin with, we are keeping it outside the regular Airbyte docker and Kube deploys for now.

Add ReadMes to better define various modules.

.bumpversion.cfg

@@ -12,18 +12,20 @@ serialize =
 
 [bumpversion:file:airbyte-bootloader/Dockerfile]
 
-[bumpversion:file:airbyte-scheduler/app/Dockerfile]
+[bumpversion:file:airbyte-container-orchestrator/Dockerfile]
 
-[bumpversion:file:airbyte-server/Dockerfile]
+[bumpversion:file:airbyte-metrics/reporter/Dockerfile]
 
-[bumpversion:file:airbyte-workers/Dockerfile]
+[bumpversion:file:airbyte-scheduler/app/Dockerfile]
 
-[bumpversion:file:airbyte-container-orchestrator/Dockerfile]
+[bumpversion:file:airbyte-server/Dockerfile]
 
 [bumpversion:file:airbyte-webapp/package.json]
 
 [bumpversion:file:airbyte-webapp/package-lock.json]
 
+[bumpversion:file:airbyte-workers/Dockerfile]
+
 [bumpversion:file:charts/airbyte/Chart.yaml]
 
 [bumpversion:file:charts/airbyte/values.yaml]

airbyte-metrics/lib/README.md

@@ -0,0 +1,3 @@
+## Airbyte Metrics Lib
+
+This module contains helpers for emitting metrics. This is primarily intended to be consumed by Airbyte Cloud, though OSS users are certainly welcome to make sure of this.

airbyte-metrics/src/main/java/io/airbyte/metrics/DogstatsdMetricSingleton.java → airbyte-metrics/lib/src/main/java/io/airbyte/metrics/lib/DogstatsdMetricSingleton.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.metrics;
+package io.airbyte.metrics.lib;
 
 import com.timgroup.statsd.NonBlockingStatsDClientBuilder;
 import com.timgroup.statsd.StatsDClient;

airbyte-metrics/reporter/Dockerfile

@@ -0,0 +1,11 @@
+ARG JDK_VERSION=17.0.1
+FROM openjdk:${JDK_VERSION}-slim AS scheduler
+
+ENV APPLICATION airbyte-metrics-reporter
+
+WORKDIR /app
+
+ADD bin/${APPLICATION}-0.35.30-alpha.tar /app
+
+# wait for upstream dependencies to become available before starting server
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.30-alpha/bin/${APPLICATION}"]

airbyte-metrics/reporter/README.md

@@ -0,0 +1,5 @@
+## Airbyte Metrics Reporter
+
+The Reporter attempts to keep code clean by centralising metric calculation and submissions.
+
+This is primarily intended for Airbyte Cloud, though OSS users are certainly welcome to make use of this.

airbyte-metrics/reporter/build.gradle

@@ -0,0 +1,27 @@
+plugins {
+    id 'application'
+}
+
+dependencies {
+    implementation project(':airbyte-metrics:lib')
+}
+
+application {
+    applicationName = "airbyte-metrics-reporter"
+    mainClass = 'io.airbyte.metrics.reporter.ReporterApp'
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+}
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-metrics-reporter-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("reporter",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.metrics.reporter;
+
+import io.airbyte.metrics.lib.DogstatsdMetricSingleton;
+
+public class ReporterApp {
+
+  public static void main(final String[] args) {
+    DogstatsdMetricSingleton.initialize("airbyte-metrics-reporter", false);
+  }
+
+}

airbyte-scheduler/app/build.gradle

@@ -12,7 +12,7 @@ dependencies {
     implementation project(':airbyte-config:persistence')
     implementation project(':airbyte-db:lib')
     implementation project(':airbyte-json-validation')
-    implementation project(':airbyte-metrics')
+    implementation project(':airbyte-metrics:lib')
     implementation project(':airbyte-protocol:models')
     implementation project(':airbyte-scheduler:models')
     implementation project(':airbyte-scheduler:persistence')

airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java

@@ -29,7 +29,7 @@
 import io.airbyte.db.Database;
 import io.airbyte.db.instance.configs.ConfigsDatabaseInstance;
 import io.airbyte.db.instance.jobs.JobsDatabaseInstance;
-import io.airbyte.metrics.DogstatsdMetricSingleton;
+import io.airbyte.metrics.lib.DogstatsdMetricSingleton;
 import io.airbyte.scheduler.models.Job;
 import io.airbyte.scheduler.models.JobStatus;
 import io.airbyte.scheduler.persistence.DefaultJobPersistence;

settings.gradle

@@ -43,7 +43,8 @@ include ':airbyte-commons-docker'
 include ':airbyte-config:models' // reused by acceptance tests in connector base.
 include ':airbyte-db:lib' // reused by acceptance tests in connector base.
 include ':airbyte-json-validation'
-include ':airbyte-metrics'
+include ':airbyte-metrics:lib'
+include ':airbyte-metrics:reporter'
 include ':airbyte-oauth'
 include ':airbyte-protocol:models'
 include ':airbyte-queue'