Add rrmt check kafka

sink-connector-lightweight/tests/integration/requirements.txt

@@ -7,4 +7,5 @@ PyYAML==5.3.1
 docker-compose==1.29.2
 awscli==1.27.36
 docker==6.1.3
-requests==2.31.0
+requests==2.31.0
+cryptography==42.0.8

sink-connector/tests/integration/configs/clickhouse/common.xml

@@ -0,0 +1,6 @@
+<clickhouse>
+    <timezone>Europe/Moscow</timezone>
+    <listen_host replace="replace">::</listen_host>
+    <path>/var/lib/clickhouse/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+</clickhouse>

sink-connector/tests/integration/configs/clickhouse/config.d/ports.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <https_port>8443</https_port>
+    <tcp_port_secure>9440</tcp_port_secure>
+    <postgresql_port>9005</postgresql_port>
+    <mysql_port>9004</mysql_port>
+</clickhouse>

sink-connector/tests/integration/configs/clickhouse/config.d/remote.xml

@@ -12,6 +12,10 @@
         <replicated_cluster>
             <shard>
                 <internal_replication>false</internal_replication>
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                </replica>
                 <replica>
                     <host>clickhouse1</host>
                     <port>9000</port>
@@ -27,6 +31,12 @@
             </shard>
         </replicated_cluster>
         <sharded_cluster>
+            <shard>
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
             <shard>
                 <replica>
                     <host>clickhouse1</host>

sink-connector/tests/integration/configs/clickhouse/config.d/ssl.xml

@@ -0,0 +1,18 @@
+<clickhouse>
+    <openSSL>
+        <server>
+            <certificateFile>/etc/clickhouse-server/ssl/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/ssl/server.key</privateKeyFile>
+            <dhParamsFile>/etc/clickhouse-server/ssl/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <cacheSessions>true</cacheSessions>
+        </server>
+        <client>
+            <cacheSessions>true</cacheSessions>
+            <verificationMode>none</verificationMode>
+            <invalidCertificateHandler>
+                <name>AcceptCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</clickhouse>

sink-connector/tests/integration/configs/clickhouse/config.d/storage.xml

@@ -0,0 +1,99 @@
+<clickhouse>
+
+<storage_configuration>
+    <disks>
+        <s3_disk>
+          <type>s3</type>
+            <endpoint>http://minio:9001/root/data/</endpoint>
+            <access_key_id>minio</access_key_id>
+            <secret_access_key>minio123</secret_access_key>
+        </s3_disk>
+        <default>
+            <keep_free_space_bytes>1024</keep_free_space_bytes>
+        </default>
+        <new_disk>
+            <path>/mnt/new_disk/</path>
+            <keep_free_space_bytes>10485760</keep_free_space_bytes>
+        </new_disk>
+        <disk1>
+            <path>/var/lib/clickhouse/disk1/</path>
+        </disk1>
+        <disk2>
+            <path>/var/lib/clickhouse/disk2/</path>
+        </disk2>
+        <jbod1>
+          <path>/jbod1/</path>
+        </jbod1>
+        <jbod2>
+          <path>/jbod2/</path>
+            <keep_free_space_bytes>10485760</keep_free_space_bytes>
+        </jbod2>
+    </disks>
+    <policies>
+      <s3_policy>
+        <volumes>
+          <main>
+            <disk>s3_disk</disk>
+          </main>
+        </volumes>
+      </s3_policy>
+        <default>
+            <volumes>
+                <default>
+                    <disk>default</disk>
+                </default>
+            </volumes>
+        </default>
+        <my_policy>
+            <volumes>
+                <main>
+                    <disk>default</disk>
+                    <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                </main>
+                <external>
+                    <disk>new_disk</disk>
+                </external>
+            </volumes>
+        </my_policy>
+        <policy1>
+            <volumes>
+                <volume2>
+                    <disk>disk1</disk>
+                </volume2>
+            </volumes>
+        </policy1>
+        <policy2>
+            <volumes>
+                <volume2>
+                    <disk>disk2</disk>
+                </volume2>
+            </volumes>
+        </policy2>
+        <tiered_storage>
+            <volumes>
+                <hot>
+                    <disk>disk1</disk>
+                </hot>
+                <cold>
+                    <disk>s3_disk</disk>
+                </cold>
+            </volumes>
+            <move_factor>0.2</move_factor>
+        </tiered_storage>
+        <fast_med_and_slow>
+          <volumes>
+              <fast>
+                  <disk>jbod1</disk>
+                </fast>
+                <medium>
+                  <disk>jbod2</disk>
+                </medium>
+                <slow>
+                  <disk>s3_disk</disk>
+                </slow>
+            </volumes>
+        </fast_med_and_slow>
+    </policies>
+</storage_configuration>
+
+</clickhouse>

sink-connector/tests/integration/configs/clickhouse/ssl/dhparam.pem

@@ -0,0 +1,8 @@
+-----BEGIN DH PARAMETERS-----
+MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR
+XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5
+Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb
+9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf
+slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb
+dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg==
+-----END DH PARAMETERS-----

sink-connector/tests/integration/configs/clickhouse/ssl/server.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow
+FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI
+4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T
+4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU
+7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj
+sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg
+pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC
+kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j
+4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr
+85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C
+L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD
++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L
+P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp
+0g==
+-----END CERTIFICATE-----

sink-connector/tests/integration/configs/clickhouse/ssl/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn
+DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L
+OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus
+TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7
+JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F
+KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6
+ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP
+TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f
+7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN
+MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5
+212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu
+plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr
+/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu
+Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa
+CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g
+1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5
+bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq
+cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez
+on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K
+GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54
+nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU
+2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh
+vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa
+dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr
+vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1
+KCBtmIrQLqzMpnBpLNuSY+Q=
+-----END PRIVATE KEY-----

sink-connector/tests/integration/configs/clickhouse0/config.d/macros.xml

@@ -1,7 +1,9 @@
 <?xml version="1.0"?>
 <clickhouse>
     <macros>
+	<cluster>replicated_cluster</cluster>
         <replica>clickhouse</replica>
         <shard>01</shard>
+        <shard2>01</shard2>
     </macros>
 </clickhouse>

sink-connector/tests/integration/configs/clickhouse1/config.d/macros.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0"?>
 <clickhouse>
     <macros>
+	    <cluster>replicated_cluster</cluster>
         <replica>clickhouse1</replica>
         <shard>01</shard>
 <!--        <shard2>01</shard2>-->

sink-connector/tests/integration/configs/clickhouse2/config.d/macros.xml

@@ -1,8 +1,9 @@
 <?xml version="1.0"?>
 <clickhouse>
     <macros>
+	    <cluster>replicated_cluster</cluster>
         <replica>clickhouse2</replica>
-        <shard>02</shard>
+        <shard>01</shard>
 <!--        <shard2>02</shard2>-->
     </macros>
 </clickhouse>

sink-connector/tests/integration/configs/clickhouse3/config.d/macros.xml

@@ -1,8 +1,8 @@
 <?xml version="1.0"?>
 <clickhouse>
     <macros>
+	    <cluster>replicated_cluster</cluster>
         <replica>clickhouse3</replica>
-        <shard>02</shard>
-<!--        <shard2>03</shard2>-->
+        <shard>01</shard>
     </macros>
 </clickhouse>

sink-connector/tests/integration/requirements.txt

@@ -7,4 +7,5 @@ PyYAML==5.3.1
 docker-compose==1.29.2
 awscli==1.27.36
 docker==6.1.3
-requests==2.31.0
+requests==2.31.0
+cryptography==42.0.8

sink-connector/tests/integration/tests/steps/service_settings_steps.py

@@ -8,6 +8,7 @@ def init_sink_connector(
     self,
     node=None,
     auto_create_tables="auto",
+    auto_create_replicated_tables=False,
     topics="SERVER5432.sbtest.sbtest1,SERVER5432.test.users1,SERVER5432.test.users2,SERVER5432.test.users3, SERVER5432.test.users",
 ):
     """
@@ -16,60 +17,16 @@ def init_sink_connector(
     if node is None:
         node = self.context.cluster.node("bash-tools")
 
+    if auto_create_replicated_tables:
+        auto_create_replicated_tables = "true"
+    else:
+        auto_create_replicated_tables = "false"
+
     if auto_create_tables == "auto":
         auto_create_tables_local = "true"
     else:
         auto_create_tables_local = "false"
 
-    sink_settings_transfer_command_apicurio = (
-        """cat <<EOF | curl --request POST --url "http://sink:8083/connectors" --header 'Content-Type: application/json' --data @-
-    {
-      "name": "sink-connector",
-      "config": {
-        "connector.class": "com.altinity.clickhouse.sink.connector.ClickHouseSinkConnector",
-        "tasks.max": "10",
-        "topics.regex": "SERVER5432.test.users, SERVER5432.sbtest.sbtest1",
-        "clickhouse.topic2table.map": "SERVER5432.test.users:users",
-        "clickhouse.server.url": "clickhouse",
-        "clickhouse.server.user": "root",
-        "clickhouse.server.password": "root",
-        "clickhouse.server.database": "test",
-        "clickhouse.server.port": 8123,
-        "clickhouse.table.name": "users",
-        "key.converter": "io.apicurio.registry.utils.converter.AvroConverter",
-        "value.converter": "io.apicurio.registry.utils.converter.AvroConverter",
-
-        "key.converter.apicurio.registry.url": "http://schemaregistry:8080/apis/registry/v2",
-        "key.converter.apicurio.registry.auto-register": "true",
-        "key.converter.apicurio.registry.find-latest": "true",
-
-        "value.converter.apicurio.registry.url": "http://schemaregistry:8080/apis/registry/v2",
-        "value.converter.apicurio.registry.auto-register": "true",
-        "value.converter.apicurio.registry.find-latest": "true",
-        "store.kafka.metadata": true,
-        "topic.creation.default.partitions": 6,
-
-        "store.raw.data": false,
-        "store.raw.data.column": "raw_data",
-
-        "metrics.enable": true,
-        "metrics.port": 8084,
-        "buffer.flush.time": 3500,
-        "thread.pool.size": 1,
-        "fetch.min.bytes": 52428800,
-
-        "enable.kafka.offset": false,
-
-        "replacingmergetree.delete.column": "_sign","""
-        + f'"auto.create.tables": {auto_create_tables_local},'
-        """
-          "schema.evolution": false,
-
-          "deduplication.policy": "off"
-        }
-      }
-EOF"""
-    )
     # "topics": "SERVER5432.test.users",
     sink_settings_transfer_command_confluent = (
         """cat <<EOF | curl --request POST --url "http://sink:8083/connectors" --header 'Content-Type: application/json' --data @-
@@ -107,7 +64,7 @@ def init_sink_connector(
     "enable.kafka.offset": false,
 
     "replacingmergetree.delete.column": "_sign","""
-        + f'"auto.create.tables": {auto_create_tables_local},'
+        + f'"auto.create.tables": {auto_create_tables_local},"auto.create.tables.replicated": {auto_create_replicated_tables},'
         """
           "schema.evolution": false,