Added initial commit of files from the original repo

.github/workflows/maven.yml

@@ -0,0 +1,28 @@
+# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+name: Java CI with Maven
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: maven
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+      - name: Run java checkstyle
+        uses: nikitasavinov/checkstyle-action@0.4.0

.gitignore

@@ -5,6 +5,8 @@
 *.so
 *.dylib
 
+target/
+
 # Test binary, build with `go test -c`
 *.test
 

config/debezium.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: debezium-connect
+  namespace: mysql
+spec:
+  containers:
+    - name: connect
+      image: debezium/connect
+      ports:
+        - name: debezium
+          containerPort: 8083
+          protocol: TCP
+      env:
+        - name: BOOTSTRAP_SERVERS
+          value: "one-node-cluster-0.one-node-cluster.redpanda.svc.cluster.local:9092"
+        - name: GROUP_ID
+          value: "1"
+        - name: CONFIG_STORAGE_TOPIC
+          value:  "inventory.configs"
+        - name: OFFSET_STORAGE_TOPIC
+          value:  "inventory.offset"
+        - name: STATUS_STORAGE_TOPIC
+          value: "inventory.status"

config/mysql.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+data:
+  rootHost: JQ==
+  rootPassword: YWx0aW5pdHk=
+  rootUser: cm9vdA==
+kind: Secret
+metadata:
+  name: mypwds
+  namespace: mysql
+type: Opaque
+
+apiVersion: mysql.oracle.com/v2alpha1
+kind: InnoDBCluster
+metadata:
+  name: mycluster
+spec:
+  secretName: mypwds
+  instances: 1
+  router:
+    instances: 1

config/redpanda.yaml

@@ -0,0 +1,29 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: one-node-cluster
+spec:
+  image: "vectorized/redpanda"
+  version: "latest"
+  replicas: 1
+  resources:
+    requests:
+      cpu: 1
+      memory: 1.2Gi
+    limits:
+      cpu: 1
+      memory: 1.2Gi
+  configuration:
+    rpcServer:
+      port: 33145
+    kafkaApi:
+      - port: 9092
+      - external:
+          enabled: true
+    pandaproxyApi:
+      - port: 8082
+      - external:
+          enabled: true
+    adminApi:
+      - port: 9644
+    developerMode: true

config/runContainers.sh

@@ -0,0 +1,33 @@
+#!/bin/zsh
+
+docker rm zookeeper
+docker rm kafka
+docker rm connect
+docker rm mysql
+
+docker run --detach --name zookeeper zookeeper &
+sleep 5
+docker run --detach --name kafka -p 8081:8081 -p 8082:8082 -p 9091:9091 -p 9092:9092 vectorized/redpanda &
+sleep 5
+docker run --detach -it --name connect -p 8083:8083 -e BOOSTRAP_SERVERS=localhost:9092 -e GROUP_ID=1 -e CONFIG_STORAGE_TOPIC=my-connect-configs -e OFFSET_STORAGE_TOPIC=my-connect-offsets -e ADVERTISED_HOST_NAME=$(echo $DOCKER_HOST | cut -f3  -d'/' | cut -f1 -d':') --link zookeeper:zookeeper --link kafka:kafka debezium/connect
+sleep 5
+
+# Activating binlog, using mariadb since mysql images are not supported in
+docker run --detach --name mysql -p 3306:3306 --env MARIADB_USER=user --env MARIADB_PASSWORD=secret --env MARIADB_ROOT_PASSWORD=root  arm64v8/mariadb:latest mysqld \
+                                                                                                                                                               --datadir=/var/lib/mysql \
+                                                                                                                                                               --user=mysql \
+                                                                                                                                                               --server-id=1 \
+                                                                                                                                                               --log-bin=/var/lib/mysql/mysql-bin.log \
+                                                                                                                                                               --binlog_do_db=test
+# Set this in mysql prompt
+# set global binlog_format = ROW;
+
+# Run clickhouse local docker image
+docker run -d -p8123:8123 --name clickhouse --ulimit nofile=262144:262144 yandex/clickhouse-server
+
+sleep 5
+
+
+# Start kafka connect standalone
+# copy containers/libs jar files to $KAFKA_CONNECT_ROOT/libs folder.
+./connect-standalone.sh ../config/connect-standalone.properties ../../GITHUB/kafka-connect-clickhouse/kcch-connector/src/main/config/mysql-debezium.properties ../../GITHUB/kafka-connect-clickhouse/kcch-connector/src/main/config/clickhouse-sink.properties

pom.xml

@@ -210,7 +210,7 @@
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
+            <!-- <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-gpg-plugin</artifactId>
                 <version>1.6</version>
@@ -223,7 +223,7 @@
                         </goals>
                     </execution>
                 </executions>
-            </plugin>
+            </plugin> -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-source-plugin</artifactId>
@@ -253,8 +253,18 @@
                     </execution>
                 </executions>
             </plugin>
+            <!-- <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>3.0.0</version>
+                <configuration>
+                    <mainClass></mainClass>
+                    <classpathScope>test</classpathScope>
+                </configuration>
+            </plugin> -->
         </plugins>
 
+        <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
         <pluginManagement>
             <plugins>
                 <plugin>
@@ -266,19 +276,20 @@
             </plugins>
         </pluginManagement>
     </build>
-
     <dependencies>
+        <!-- https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-fips -->
         <dependency>
             <groupId>org.bouncycastle</groupId>
             <artifactId>bcpkix-fips</artifactId>
             <version>1.0.3</version>
             <scope>provided</scope>
         </dependency>
 
+        <!-- https://mvnrepository.com/artifact/org.apache.kafka/connect-api -->
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>connect-api</artifactId>
-            <version>3.1.0</version>
+            <version>2.8.1</version>
             <scope>provided</scope>
             <exclusions>
                 <exclusion>
@@ -327,19 +338,16 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
             <version>2.12.6</version>
         </dependency>
-
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
             <version>2.12.6</version>
         </dependency>
-
         <dependency>
             <groupId>io.confluent</groupId>
             <artifactId>kafka-schema-registry-client</artifactId>
@@ -363,7 +371,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
         <dependency>
             <groupId>io.confluent</groupId>
             <artifactId>kafka-avro-serializer</artifactId>
@@ -375,14 +382,29 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
+        <!-- https://mvnrepository.com/artifact/io.confluent/kafka-connect-avro-converter -->
         <dependency>
             <groupId>io.confluent</groupId>
             <artifactId>kafka-connect-avro-converter</artifactId>
             <version>5.5.1</version>
             <scope>test</scope>
         </dependency>
 
+        <!--JDBC driver for building connection with Clickhouse-->
+        <!-- <dependency>
+            <groupId>ru.yandex.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.3.2</version>
+        </dependency> -->
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <!-- or clickhouse-grpc-client if you prefer gRPC -->
+            <artifactId>clickhouse-http-client</artifactId>
+            <version>0.3.2-patch7</version>
+        </dependency>
+
+
+        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-core</artifactId>
@@ -401,66 +423,57 @@
             <version>31.0.1-jre</version>
         </dependency>
 
-        <dependency>
-            <groupId>dev.failsafe</groupId>
-            <artifactId>failsafe</artifactId>
-            <version>3.2.1</version>
-        </dependency>
-
+        <!--junit for unit test-->
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
             <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
 
+        <!--Mockito for unit test-->
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
             <version>2.20.1</version>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
             <version>1.7.25</version>
         </dependency>
-
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
             <version>1.7.25</version>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>
             <version>2.17.1</version>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>com.github.stefanbirkner</groupId>
             <artifactId>system-rules</artifactId>
             <version>1.19.0</version>
             <scope>test</scope>
         </dependency>
 
+        <!--Kafka JSON converter for SMT unit test-->
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>connect-json</artifactId>
-            <version>0.9.0.0</version>
+            <version>3.1.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.clickhouse</groupId>
-            <!-- or clickhouse-grpc-client if you prefer gRPC -->
-            <artifactId>clickhouse-http-client</artifactId>
-            <version>0.3.2-patch7</version>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>connect-json</artifactId>
+            <version>3.1.0</version>
+            <scope>compile</scope>
         </dependency>
     </dependencies>
-
 </project>
-