<refactor>[ingestions]: align the default kafka topics with PR #1756 (#1758)

Author: RealChrisL

contrib/metadata-ingestion/bin/dataset-hive-generator.py

@@ -11,7 +11,7 @@
 HIVESTORE='localhost'
 
 AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
-KAFKATOPIC = 'MetadataChangeEvent'
+KAFKATOPIC = 'MetadataChangeEvent_v4'
 BOOTSTRAP = 'localhost:9092'
 SCHEMAREGISTRY = 'http://localhost:8081'
 

contrib/metadata-ingestion/openldap-etl/openldap-etl.py

@@ -15,7 +15,7 @@
 SEARCHFILTER='givenname=Homer'
 
 AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
-KAFKATOPIC = 'MetadataChangeEvent'
+KAFKATOPIC = 'MetadataChangeEvent_v4'
 BOOTSTRAP = 'localhost:9092'
 SCHEMAREGISTRY = 'http://localhost:8081'
 
@@ -161,4 +161,4 @@ def produce_corp_user_mce(mce):
         break
 
 l.unbind()
-sys.exit(0)
+sys.exit(0)

docker/kafka/Dockerfile

@@ -3,6 +3,6 @@ FROM confluentinc/cp-kafka:5.4.0
 
 CMD echo Waiting for Kafka to be ready... && \
     cub kafka-ready -b $KAFKA_BOOTSTRAP_SERVER 1 60 && \
-    kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataAuditEvent && \
-    kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataChangeEvent && \
-    kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic FailedMetadataChangeEvent
+    kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataAuditEvent_v4 && \
+    kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataChangeEvent_v4 && \
+    kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic FailedMetadataChangeEvent_v4

metadata-ingestion/kafka-etl/kafka_etl.py

@@ -8,7 +8,7 @@
 
 ZOOKEEPER='localhost:2181'
 AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
-KAFKATOPIC = 'MetadataChangeEvent'
+KAFKATOPIC = 'MetadataChangeEvent_v4'
 BOOTSTRAP = 'localhost:9092'
 SCHEMAREGISTRY = 'http://localhost:8081'
 
@@ -64,4 +64,4 @@ def produce_kafka_dataset_mce(mce):
     print(topic)
     build_kafka_dataset_mce(dataset_name, str(schema), int(schema_version))
 
-sys.exit(0)
+sys.exit(0)

metadata-ingestion/ldap-etl/ldap_etl.py

@@ -17,7 +17,7 @@
 SEARCHFILTER='SEARCHFILTER'
 
 AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
-KAFKATOPIC = 'MetadataChangeEvent'
+KAFKATOPIC = 'MetadataChangeEvent_v4'
 BOOTSTRAP = 'localhost:9092'
 SCHEMAREGISTRY = 'http://localhost:8081'
 
@@ -154,4 +154,4 @@ def produce_corp_user_mce(mce):
         break
 
 l.unbind()
-sys.exit(0)
+sys.exit(0)

metadata-ingestion/mce-cli/mce_cli.py

@@ -7,7 +7,7 @@
 from confluent_kafka.avro.serializer import SerializerError
 
 
-topic = "MetadataChangeEvent"
+topic = "MetadataChangeEvent_v4"
 
 class MetadataChangeEvent(object):
 

metadata-ingestion/mysql-etl/mysql_etl.py

@@ -12,7 +12,7 @@
 PASSWORD = 'PASSWORD'
 
 AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
-KAFKATOPIC = 'MetadataChangeEvent'
+KAFKATOPIC = 'MetadataChangeEvent_v4'
 BOOTSTRAP = 'localhost:9092'
 SCHEMAREGISTRY = 'http://localhost:8081'
 
@@ -77,4 +77,4 @@ def produce_mysql_dataset_mce(mce):
 except Error as e:
     sys.stdout.write('Error while connecting to MySQL %s' % e)
 
-sys.exit(0)
+sys.exit(0)

metadata-ingestion/sql-etl/common.py

@@ -15,7 +15,7 @@
 @dataclass
 class KafkaConfig:
     avsc_path = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
-    kafka_topic = 'MetadataChangeEvent'
+    kafka_topic = 'MetadataChangeEvent_v4'
     bootstrap_server = 'localhost:9092'
     schema_registry = 'http://localhost:8081'