feat(gms): Merge MAE, MCE consumers into GMS (#2690)

datahub-kubernetes/datahub/Chart.yaml

@@ -20,11 +20,11 @@ dependencies:
   - name: datahub-mae-consumer
     version: 0.2.1
     repository: file://./charts/datahub-mae-consumer
-    condition: datahub-mae-consumer.enabled
+    condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
     version: 0.2.1
     repository: file://./charts/datahub-mce-consumer
-    condition: datahub-mce-consumer.enabled
+    condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron
     version: 0.2.1
     repository: file://./charts/datahub-ingestion-cron

datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml

@@ -73,6 +73,14 @@ spec:
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           env:
+            {{- if not .Values.global.datahub_standalone_consumers_enabled }}
+            - name: MCE_CONSUMER_ENABLED
+              value: "true"
+            - name: MAE_CONSUMER_ENABLED
+              value: "true"
+            {{- end }}
+            - name: DATAHUB_ANALYTICS_ENABLED
+              value: "{{ .Values.global.datahub_analytics_enabled }}"
             - name: EBEAN_DATASOURCE_USERNAME
               value: "{{ .Values.global.sql.datasource.username }}"
             - name: EBEAN_DATASOURCE_PASSWORD

datahub-kubernetes/datahub/charts/datahub-gms/values.yaml

@@ -146,6 +146,8 @@ readinessProbe:
 #This section is useful if we are installing this chart separately for testing
 # helm install datahub-gms datahub-gms/
 global:
+  datahub_analytics_enabled: true
+
   elasticsearch:
     host: "elasticsearch"
     port: "9200"

datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml

@@ -73,6 +73,8 @@ spec:
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           env:
+            - name: MAE_CONSUMER_ENABLED
+              value: "true"
             - name: GMS_HOST
               value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
             - name: GMS_PORT

datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml

@@ -69,6 +69,8 @@ spec:
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           env:
+            - name: MCE_CONSUMER_ENABLED
+              value: "true"
             - name: KAFKA_BOOTSTRAP_SERVER
               value: "{{ .Values.global.kafka.bootstrap.server }}"
             - name: KAFKA_SCHEMAREGISTRY_URL

datahub-kubernetes/datahub/quickstart-values.yaml

@@ -15,18 +15,6 @@ datahub-frontend:
   ingress:
     enabled: false
 
-datahub-mae-consumer:
-  enabled: true
-  image:
-    repository: linkedin/datahub-mae-consumer
-    tag: "v0.8.1"
-
-datahub-mce-consumer:
-  enabled: true
-  image:
-    repository: linkedin/datahub-mce-consumer
-    tag: "v0.8.1"
-
 elasticsearchSetupJob:
   enabled: true
   image:

datahub-kubernetes/datahub/values.yaml

@@ -58,7 +58,9 @@ datahubUpgrade:
     tag: "v0.8.1"
 
 global:
+
   datahub_analytics_enabled: true
+  datahub_standalone_consumers_enabled: false
 
   elasticsearch:
     host: "elasticsearch-master"

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java

@@ -58,7 +58,6 @@ private List<UpgradeStep> buildUpgradeSteps(
     final List<UpgradeStep> steps = new ArrayList<>();
     steps.add(new RemoveAspectV2TableStep(server));
     steps.add(new GMSQualificationStep());
-    steps.add(new MAEQualificationStep());
     steps.add(new UpgradeQualificationStep(server));
     steps.add(new CreateAspectTableStep(server));
     steps.add(new IngestDataPlatformsStep(entityService));

docker/datahub-gms/env/docker.env

@@ -13,6 +13,12 @@ NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
 
+MAE_CONSUMER_ENABLED=true
+MCE_CONSUMER_ENABLED=true
+
+# Uncomment to disable persistence of client-side analytics events
+# DATAHUB_ANALYTICS_ENABLED=false
+
 # Uncomment to configure kafka topic names
 # Make sure these names are consistent across the whole deployment
 # METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4

docker/datahub-gms/env/docker.mariadb.env

@@ -11,3 +11,5 @@ NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
+MAE_CONSUMER_ENABLED=true
+MCE_CONSUMER_ENABLED=true

docker/datahub-gms/env/docker.postgres.env

@@ -11,3 +11,6 @@ NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
+MAE_CONSUMER_ENABLED=true
+MCE_CONSUMER_ENABLED=true
+

docker/datahub-mae-consumer/env/docker.env

@@ -1,3 +1,4 @@
+MAE_CONSUMER_ENABLED=true
 KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch

docker/datahub-mce-consumer/env/docker.env

@@ -1,3 +1,4 @@
+MCE_CONSUMER_ENABLED=true
 KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 GMS_HOST=datahub-gms

docker/docker-compose.consumers.dev.yml

@@ -0,0 +1,23 @@
+version: '3.8'
+services:
+  datahub-mae-consumer:
+    image: linkedin/datahub-mae-consumer:debug
+    build:
+      context: datahub-mae-consumer
+      dockerfile: Dockerfile
+      args:
+        APP_ENV: dev
+    volumes:
+      - ./datahub-mae-consumer/start.sh:/datahub/datahub-mae-consumer/scripts/start.sh
+      - ../metadata-jobs/mae-consumer-job/build/libs/:/datahub/datahub-mae-consumer/bin/
+
+  datahub-mce-consumer:
+    image: linkedin/datahub-mce-consumer:debug
+    build:
+      context: datahub-mce-consumer
+      dockerfile: Dockerfile
+      args:
+        APP_ENV: dev
+    volumes:
+      - ./datahub-mce-consumer/start.sh:/datahub/datahub-mce-consumer/scripts/start.sh
+      - ../metadata-jobs/mce-consumer-job/build/libs/:/datahub/datahub-mce-consumer/bin

docker/docker-compose.consumers.yml

@@ -0,0 +1,31 @@
+# Service definitions for standalone Kafka consumer containers.
+version: '3.8'
+services:
+  datahub-mae-consumer:
+    build:
+      context: ../
+      dockerfile: docker/datahub-mae-consumer/Dockerfile
+    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest}
+    env_file: datahub-mae-consumer/env/docker.env
+    hostname: datahub-mae-consumer
+    container_name: datahub-mae-consumer
+    ports:
+      - "9091:9091"
+    depends_on:
+      - kafka-setup
+      - elasticsearch-setup
+      - neo4j
+
+  datahub-mce-consumer:
+    build:
+      context: ../
+      dockerfile: docker/datahub-mce-consumer/Dockerfile
+    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest}
+    env_file: datahub-mce-consumer/env/docker.env
+    hostname: datahub-mce-consumer
+    container_name: datahub-mce-consumer
+    ports:
+      - "9090:9090"
+    depends_on:
+      - kafka-setup
+      - datahub-gms

docker/docker-compose.dev.yml

@@ -50,25 +50,3 @@ services:
         APP_ENV: dev
     volumes:
       - ../datahub-frontend/build/stage/datahub-frontend:/datahub-frontend
-
-  datahub-mae-consumer:
-    image: linkedin/datahub-mae-consumer:debug
-    build:
-      context: datahub-mae-consumer
-      dockerfile: Dockerfile
-      args:
-        APP_ENV: dev
-    volumes:
-      - ./datahub-mae-consumer/start.sh:/datahub/datahub-mae-consumer/scripts/start.sh
-      - ../metadata-jobs/mae-consumer-job/build/libs/:/datahub/datahub-mae-consumer/bin/
-
-  datahub-mce-consumer:
-    image: linkedin/datahub-mce-consumer:debug
-    build:
-      context: datahub-mce-consumer
-      dockerfile: Dockerfile
-      args:
-        APP_ENV: dev
-    volumes:
-      - ./datahub-mce-consumer/start.sh:/datahub/datahub-mce-consumer/scripts/start.sh
-      - ../metadata-jobs/mce-consumer-job/build/libs/:/datahub/datahub-mce-consumer/bin

docker/docker-compose.yml

@@ -160,35 +160,6 @@ services:
     depends_on:
       - datahub-gms
 
-  datahub-mae-consumer:
-    build:
-      context: ../
-      dockerfile: docker/datahub-mae-consumer/Dockerfile
-    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest}
-    env_file: datahub-mae-consumer/env/docker.env
-    hostname: datahub-mae-consumer
-    container_name: datahub-mae-consumer
-    ports:
-        - "9091:9091"
-    depends_on:
-      - kafka-setup
-      - elasticsearch-setup
-      - neo4j
-
-  datahub-mce-consumer:
-    build:
-      context: ../
-      dockerfile: docker/datahub-mce-consumer/Dockerfile
-    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest}
-    env_file: datahub-mce-consumer/env/docker.env
-    hostname: datahub-mce-consumer
-    container_name: datahub-mce-consumer
-    ports:
-      - "9090:9090"
-    depends_on:
-      - kafka-setup
-      - datahub-gms
-
 networks:
   default:
     name: datahub_network

docker/quickstart/docker-compose.quickstart.yml

@@ -55,48 +55,13 @@ services:
     - NEO4J_URI=bolt://neo4j
     - NEO4J_USERNAME=neo4j
     - NEO4J_PASSWORD=datahub
+    - MAE_CONSUMER_ENABLED=true
+    - MCE_CONSUMER_ENABLED=true
     hostname: datahub-gms
     image: linkedin/datahub-gms:${DATAHUB_VERSION:-latest}
     mem_limit: 850m
     ports:
     - 8080:8080
-  datahub-mae-consumer:
-    container_name: datahub-mae-consumer
-    depends_on:
-    - kafka-setup
-    - elasticsearch-setup
-    - neo4j
-    environment:
-    - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
-    - ELASTICSEARCH_HOST=elasticsearch
-    - ELASTICSEARCH_PORT=9200
-    - NEO4J_HOST=http://neo4j:7474
-    - NEO4J_URI=bolt://neo4j
-    - NEO4J_USERNAME=neo4j
-    - NEO4J_PASSWORD=datahub
-    - GMS_HOST=datahub-gms
-    - GMS_PORT=8080
-    hostname: datahub-mae-consumer
-    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest}
-    mem_limit: 256m
-    ports:
-    - 9091:9091
-  datahub-mce-consumer:
-    container_name: datahub-mce-consumer
-    depends_on:
-    - kafka-setup
-    - datahub-gms
-    environment:
-    - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
-    - GMS_HOST=datahub-gms
-    - GMS_PORT=8080
-    hostname: datahub-mce-consumer
-    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest}
-    mem_limit: 384m
-    ports:
-    - 9090:9090
   elasticsearch:
     container_name: elasticsearch
     environment:

gms/war/build.gradle

@@ -4,6 +4,8 @@ ext.apiProject = project(':gms:api')
 
 dependencies {
   runtime project(':gms:factories')
+  runtime project(':metadata-jobs:mce-consumer')
+  runtime project(':metadata-jobs:mae-consumer')
 
   runtime externalDependency.h2
   runtime externalDependency.logbackClassic