Rework docker-compose-mysql.yml example used in the quikcstart guide

doc/quickstart.md

@@ -31,7 +31,7 @@ sudo apt install clickhouse-client
 Use Docker Compose to start containers. 
 ```
 cd sink-connector-lightweight/docker
-export SINK_LIGHTWEIGHT_VERSION=latest
+export CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE=latest
 docker compose -f docker-compose-mysql.yml up --renew-anon-volumes
 ```
 

sink-connector-lightweight/docker/clickhouse-service.yml

@@ -0,0 +1,21 @@
+version: "3.4"
+
+services:
+  clickhouse:
+    container_name: clickhouse
+    image: clickhouse/clickhouse-server:latest
+    restart: "no"
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    environment:
+      - CLICKHOUSE_USER=root
+      - CLICKHOUSE_PASSWORD=root
+      - CLICKHOUSE_DB=test
+      - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=0
+    ulimits:
+      nofile:
+        soft: "262144"
+        hard: "262144"
+    volumes:
+      - ../clickhouse/users.xml:/etc/clickhouse-server/users.xml

sink-connector-lightweight/docker/clickhouse-sink-connector-lt-service.yml

@@ -0,0 +1,15 @@
+version: "3.4"
+
+services:
+  clickhouse-sink-connector-lt:
+    image: ${CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE}
+    entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]
+    restart: "no"
+    ports:
+      - "8083:8083"
+      - "5005:5005"
+      - "7000:7000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./config.yml:/config.yml

sink-connector-lightweight/docker/docker-compose-mariadb.yml

@@ -8,102 +8,42 @@ version: "3.4"
 
 services:
   mysql-master:
-    container_name: mysql-master
-    command: --transaction-isolation=READ-COMMITTED --log-bin=binlog --binlog-format=ROW
-    image: mariadb
-    restart: "no"
-    ports:
-      - "3306:3306"
-    environment:
-      - MYSQL_ROOT_PASSWORD=root
-      - MYSQL_DATABASE=test
-      - MYSQL_REPLICATION_MODE=master
-      - MYSQL_REPLICATION_USER=repl_user
-      - ALLOW_EMPTY_PASSWORD=yes
-      - MYSQL_BINLOG_FORMAT=row
-    volumes:
-      - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
-      - ../sql/init_mariadb.sql:/docker-entrypoint-initdb.d/init_mysql.sql
-    healthcheck:
-      test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ]
-      interval: 15s
-      timeout: 5s
-      retries: 6
+    extends:
+      file: mysql-master-service.yml
+      service: mysql-master
 
   clickhouse:
-    container_name: clickhouse
-    image: clickhouse/clickhouse-server:latest
-    restart: "no"
-    ports:
-      - "8123:8123"
-      - "9000:9000"
-    environment:
-      - CLICKHOUSE_USER=root
-      - CLICKHOUSE_PASSWORD=root
-      - CLICKHOUSE_DB=test
-      - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=0
-    ulimits:
-      nofile:
-        soft: "262144"
-        hard: "262144"
-    volumes:
-      -  ../clickhouse/users.xml:/etc/clickhouse-server/users.xml
+    extends:
+      file: clickhouse-service.yml
+      service: clickhouse
     depends_on:
       zookeeper:
         condition: service_healthy
 
   zookeeper:
-    image: zookeeper:3.6.2
-    expose:
-      - "2181"
-    environment:
-      ZOO_TICK_TIME: 500
-      ZOO_MY_ID: 1
-    healthcheck:
-      test: echo stat | nc localhost 2181
-      interval: 3s
-      timeout: 2s
-      retries: 5
-      start_period: 2s
-    security_opt:
-      - label:disable
+    extends:
+      file: zookeeper-service.yml
+      service: zookeeper
 
-  debezium-embedded:
-    image: registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:${SINK_LIGHTWEIGHT_VERSION}
-    entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]
-    restart: "no"
-    ports:
-      - "8083:8083"
-      - "5005:5005"
-      - "7000:7000"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - ./config.yml:/config.yml
+  clickhouse-sink-connector-lt:
+    extends:
+      file: clickhouse-sink-connector-lt-service.yml
+      service: clickhouse-sink-connector-lt
 
   ### MONITORING ####
   prometheus:
-    container_name: prometheus
-    image: bitnami/prometheus:2.36.0
-    restart: "no"
-    ports:
-      - "9090:9090"
-    volumes:
-      - ./config/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml
+    extends:
+      file: prometheus-service.yml
+      service: prometheus
 
   grafana:
-    build:
-      context: ./config/grafana
-      args:
-        GRAFANA_VERSION: latest
-    restart: "no"
-    ports:
-      - "3000:3000"
-    environment:
-      - DS_PROMETHEUS=prometheus
-      - GF_USERS_DEFAULT_THEME=light
-      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
-      - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
+    extends:
+      file: grafana-service.yml
+      service: grafana
+    volumes:
+      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
     depends_on:
       - prometheus
     ## END OF MONITORING ###

sink-connector-lightweight/docker/docker-compose-mysql-standalone.yml

@@ -1,39 +1,23 @@
 services:
-  debezium-embedded:
-    image: registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:${SINK_LIGHTWEIGHT_VERSION}
-    entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]
-    restart: "no"
-    ports:
-      - "8083:8083"
-      - "5005:5005"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - ./config.yml:/config.yml
+  clickhouse-sink-connector-lt:
+    extends:
+      file: clickhouse-sink-connector-lt-service.yml
+      service: clickhouse-sink-connector-lt
 
   ### MONITORING ####
   prometheus:
-    container_name: prometheus
-    image: bitnami/prometheus:2.36.0
-    restart: "no"
-    ports:
-      - "9090:9090"
-    volumes:
-      - ./config/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml
+    extends:
+      file: prometheus-service.yml
+      service: prometheus
 
   grafana:
-    build:
-      context: ./config/grafana
-      args:
-        GRAFANA_VERSION: latest
-    restart: "no"
-    ports:
-      - "3000:3000"
-    environment:
-      - DS_PROMETHEUS=prometheus
-      - GF_USERS_DEFAULT_THEME=light
-      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
-      - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
+    extends:
+      file: grafana-service.yml
+      service: grafana
+    volumes:
+      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
     depends_on:
       - prometheus
     ## END OF MONITORING ###

sink-connector-lightweight/docker/docker-compose-mysql.yml

@@ -8,101 +8,44 @@ version: "3.4"
 
 services:
   mysql-master:
-    container_name: mysql-master
-    image: docker.io/bitnami/mysql:latest
-    restart: "no"
-    ports:
-      - "3306:3306"
-    environment:
-      - MYSQL_ROOT_PASSWORD=root
-      - MYSQL_DATABASE=test
-      - MYSQL_REPLICATION_MODE=master
-      - MYSQL_REPLICATION_USER=repl_user
-      - ALLOW_EMPTY_PASSWORD=yes
-    volumes:
-      - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
-      - ../sql/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql
-    healthcheck:
-      test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ]
-      interval: 15s
-      timeout: 5s
-      retries: 6
+    extends:
+      file: mysql-master-service.yml
+      service: mysql-master
 
   clickhouse:
-    container_name: clickhouse
-    image: clickhouse/clickhouse-server:latest
-    restart: "no"
-    ports:
-      - "8123:8123"
-      - "9000:9000"
-    environment:
-      - CLICKHOUSE_USER=root
-      - CLICKHOUSE_PASSWORD=root
-      - CLICKHOUSE_DB=test
-      - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=0
-    ulimits:
-      nofile:
-        soft: "262144"
-        hard: "262144"
-    volumes:
-      -  ../clickhouse/users.xml:/etc/clickhouse-server/users.xml
+    extends:
+      file: clickhouse-service.yml
+      service: clickhouse
     depends_on:
       zookeeper:
         condition: service_healthy
 
   zookeeper:
-      image: zookeeper:3.6.2
-      expose:
-        - "2181"
-      environment:
-        ZOO_TICK_TIME: 500
-        ZOO_MY_ID: 1
-      healthcheck:
-        test: echo stat | nc localhost 2181
-        interval: 3s
-        timeout: 2s
-        retries: 5
-        start_period: 2s
-      security_opt:
-        - label:disable
+    extends:
+      file: zookeeper-service.yml
+      service: zookeeper
 
-  debezium-embedded:
-    image: altinityinfra/clickhouse-sink-connector:409-99c65c73fb059e0509b2ec7b33cfb414d3585b56-lt
-    entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]
-    restart: "no"
-    ports:
-      - "8083:8083"
-      - "5005:5005"
-      - "7000:7000"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - ./config.yml:/config.yml
+  clickhouse-sink-connector-lt:
+    extends:
+      file: clickhouse-sink-connector-lt-service.yml
+      service: clickhouse-sink-connector-lt
 
 
   ### MONITORING ####
   prometheus:
-    container_name: prometheus
-    image: bitnami/prometheus:2.36.0
-    restart: "no"
-    ports:
-      - "9090:9090"
-    volumes:
-      - ./config/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml
+    extends:
+      file: prometheus-service.yml
+      service: prometheus
 
   grafana:
-    build:
-      context: ./config/grafana
-      args:
-        GRAFANA_VERSION: latest
-    restart: "no"
-    ports:
-      - "3000:3000"
-    environment:
-      - DS_PROMETHEUS=prometheus
-      - GF_USERS_DEFAULT_THEME=light
-      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
-      - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
+    extends:
+      file: grafana-service.yml
+      service: grafana
+    volumes:
+      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
     depends_on:
       - prometheus
+
     ## END OF MONITORING ###