Merge pull request #621 from Altinity/fix_runners_24_4

Fix runners 24 4

.github/workflows/testflows-sink-connector-kafka.yml

@@ -48,6 +48,18 @@ jobs:
         working-directory: sink-connector/tests/integration
         run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
 
+      - name: Create a virtual environment
+        run: |
+          echo "Install Python modules..."
+          sudo apt-get clean
+          sudo apt-get update
+          sudo apt-get install -y python3.12-venv
+
+          echo "Create and activate Python virtual environment..."
+          python3 -m venv venv
+          source venv/bin/activate
+          echo PATH=$PATH >> $GITHUB_ENV
+
       - name: Install all dependencies
         working-directory: sink-connector/tests/integration
         run: pip3 install -r requirements.txt

.github/workflows/testflows-sink-connector-lightweight.yml

@@ -64,6 +64,18 @@ jobs:
         working-directory: sink-connector/tests/integration
         run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
 
+      - name: Create a virtual environment
+        run: |
+          echo "Install Python modules..."
+          sudo apt-get clean
+          sudo apt-get update
+          sudo apt-get install -y python3.12-venv
+          
+          echo "Create and activate Python virtual environment..."
+          python3 -m venv venv
+          source venv/bin/activate
+          echo PATH=$PATH >> $GITHUB_ENV
+
       - name: Install all dependencies
         working-directory: sink-connector-lightweight/tests/integration
         run: pip3 install -r requirements.txt
@@ -130,6 +142,18 @@ jobs:
         working-directory: sink-connector/tests/integration
         run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
 
+      - name: Create a virtual environment
+        run: |
+          echo "Install Python modules..."
+          sudo apt-get clean
+          sudo apt-get update
+          sudo apt-get install -y python3.12-venv
+
+          echo "Create and activate Python virtual environment..."
+          python3 -m venv venv
+          source venv/bin/activate
+          echo PATH=$PATH >> $GITHUB_ENV
+
       - name: Install all dependencies
         working-directory: sink-connector-lightweight/tests/integration
         run: pip3 install -r requirements.txt

doc/quickstart.md

@@ -31,7 +31,7 @@ sudo apt install clickhouse-client
 Use Docker Compose to start containers. 
 ```
 cd sink-connector-lightweight/docker
-export CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE=altinityinfra/clickhouse-sink-connector:408-97b1d3d83ef93c1b76a2b1c4d9c544dc67fbbec3-lt
+export CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE=altinity/clickhouse-sink-connector:2.1.0-lt
 docker compose -f docker-compose-mysql.yml up --renew-anon-volumes
 ```
 
@@ -139,4 +139,4 @@ schema.history.internal.jdbc.url: "jdbc:clickhouse://cloud_url:8443/altinity_sin
 ## References:
 1. [Sink Connector Configuration ](configuration.md)
 2. [MySQL Topologies supported](https://debezium.io/documentation/reference/2.5/connectors/mysql.html#setting-up-mysql)
-3. [MySQL Setup](https://debezium.io/documentation/reference/2.5/connectors/mysql.html#setting-up-mysql)
+3. [MySQL Setup](https://debezium.io/documentation/reference/2.5/connectors/mysql.html#setting-up-mysql)

sink-connector-lightweight/tests/integration/requirements.txt

@@ -1,5 +1,10 @@
-docker-compose==1.29.2
 testflows==2.1.5
+python-dateutil==2.9.0
+numpy==1.26.4
+pyarrow==16.1.0
+pandas==2.2.0
+PyYAML==5.3.1
+docker-compose==1.29.2
 awscli==1.27.36
 docker==6.1.3
-requests==2.31.0
+requests==2.31.0

sink-connector-lightweight/tests/integration/tests/calculated_columns.py

@@ -34,7 +34,7 @@ def string_concatenation(self):
         f"I create a {table_name} table with calculated column with string concatenation"
     ):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"first_name VARCHAR(50) NOT NULL,last_name VARCHAR(50) NOT NULL,fullname varchar(101) "
             f"GENERATED ALWAYS AS (CONCAT(first_name,' ',last_name)),email VARCHAR(100) NOT NULL",
         )
@@ -60,7 +60,7 @@ def basic_arithmetic_operations(self):
 
     with Given(f"I create a {table_name} table with calculated column"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"a INT, b INT, sum_col INT AS (a + b), diff_col INT AS (a - b), prod_col INT AS (a * b), div_col DOUBLE AS (a / b)",
         )
 
@@ -86,7 +86,7 @@ def complex_expressions(self):
 
     with Given(f"I create a {table_name} table with calculated column"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"base_salary DECIMAL(10,2), bonus_rate DECIMAL(5,2), total_compensation DECIMAL(12,2) AS (base_salary + (base_salary * bonus_rate / 100))",
         )
 
@@ -114,7 +114,7 @@ def nested(self):
 
     with Given(f"I create a {table_name} table with calculated column"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"a INT, b INT, c INT AS (a + b), d INT AS (c * 2)",
         )
 

sink-connector-lightweight/tests/integration/tests/datatypes.py

@@ -36,7 +36,7 @@ def create_table_with_datetime_column(self, table_name, data, precision):
 
     with By(f"creating a {table_name} table with datetime column"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"date DATETIME({precision})",
         )
 

sink-connector-lightweight/tests/integration/tests/is_deleted.py

@@ -19,7 +19,7 @@ def create_table_with_is_deleted(
         f"creating a {table_name} table with is_deleted column and {datatype} datatype"
     ):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"col1 varchar(255), col2 int, {column} {datatype}",
         )
 

sink-connector-lightweight/tests/integration/tests/multiple_databases.py

@@ -113,7 +113,7 @@ def create_table_and_insert_values(
 
     with By("creating a sample table in MySQL"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"col1 varchar(255), col2 int",
             database_name=database_name,
         )

sink-connector-lightweight/tests/integration/tests/retry_on_fail.py

@@ -16,7 +16,7 @@ def retry_on_fail(self):
 
     with When("I creat a table in MySQL"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"retry VARCHAR(16)",
         )
 

sink-connector-lightweight/tests/integration/tests/schema_only.py

@@ -11,7 +11,7 @@ def create_table_structure(self, table_name):
 
     with By(f"creating a {table_name} table"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"col1 varchar(255), col2 int",
         )
 

sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py

@@ -47,7 +47,7 @@ def create_and_validate_table(self, table_name):
         "creating a table in MySQL and checking that it was also created in ClickHouse"
     ):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns=f"col1 varchar(255), col2 int",
         )
 

sink-connector-lightweight/tests/integration/tests/steps/alter.py

@@ -20,7 +20,7 @@ def add_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type};"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type};"
     )
 
 
@@ -41,7 +41,7 @@ def rename_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` RENAME COLUMN {column_name} to {new_column_name};"
+        rf"ALTER TABLE {database}.\`{table_name}\` RENAME COLUMN {column_name} to {new_column_name};"
     )
 
 
@@ -63,7 +63,7 @@ def change_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` CHANGE COLUMN {column_name} {new_column_name} {new_column_type};"
+        rf"ALTER TABLE {database}.\`{table_name}\` CHANGE COLUMN {column_name} {new_column_name} {new_column_type};"
     )
 
 
@@ -84,7 +84,7 @@ def modify_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` MODIFY COLUMN {column_name} {new_column_type};"
+        rf"ALTER TABLE {database}.\`{table_name}\` MODIFY COLUMN {column_name} {new_column_type};"
     )
 
 
@@ -97,7 +97,7 @@ def drop_column(self, table_name, column_name="new_col", node=None, database=Non
     if node is None:
         node = self.context.cluster.node("mysql-master")
 
-    node.query(f"ALTER TABLE {database}.\`{table_name}\` DROP COLUMN {column_name};")
+    node.query(rf"ALTER TABLE {database}.\`{table_name}\` DROP COLUMN {column_name};")
 
 
 @TestStep(When)
@@ -149,7 +149,7 @@ def add_column_null_not_null(
 
     null_not_null = "NOT NULL" if not is_null else "NULL"
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} {null_not_null};"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} {null_not_null};"
     )
 
 
@@ -171,7 +171,7 @@ def add_column_default(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} DEFAULT {default_value};"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} DEFAULT {default_value};"
     )
 
 
@@ -185,5 +185,5 @@ def add_primary_key(self, table_name, column_name, node=None, database=None):
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD PRIMARY KEY ({column_name});"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD PRIMARY KEY ({column_name});"
     )

sink-connector-lightweight/tests/integration/tests/steps/mysql.py

@@ -324,7 +324,7 @@ def insert(self, table_name, values, node=None, database_name=None):
         node = self.context.cluster.node("mysql-master")
 
     with When("I insert data into MySQL table"):
-        node.query(f"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});")
+        node.query(rf"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});")
 
 
 @TestStep(Given)

sink-connector-lightweight/tests/integration/tests/table_names.py

@@ -52,23 +52,23 @@ def check_table_names(self, table_name):
 
     with Given(f"I create the {table_name} table"):
         create_mysql_table(
-            table_name=f"\`{table_name}\`",
+            table_name=rf"\`{table_name}\`",
             columns="x INT",
         )
 
     with And("I insert data into the table"):
-        mysql_node.query(f"INSERT INTO \`{table_name}\` VALUES (1, 1);")
+        mysql_node.query(rf"INSERT INTO \`{table_name}\` VALUES (1, 1);")
 
     with Then(f"I check that the {table_name} was created in the ClickHouse side"):
         for retry in retries(timeout=40, delay=1):
             with retry:
-                clickhouse_node.query(f"EXISTS test.\`{table_name}\`", message="1")
+                clickhouse_node.query(rf"EXISTS test.\`{table_name}\`", message="1")
 
     with And("I check that the data was inserted correctly into the ClickHouse table"):
         for retry in retries(timeout=40, delay=1):
             with retry:
                 clickhouse_data = clickhouse_node.query(
-                    f"SELECT id,x FROM test.\`{table_name}\` FORMAT CSV"
+                    rf"SELECT id,x FROM test.\`{table_name}\` FORMAT CSV"
                 )
                 assert clickhouse_data.output.strip() == "1,1", error()
 

sink-connector/tests/integration/requirements.txt

@@ -1,5 +1,10 @@
-docker-compose==1.29.2
 testflows==2.1.5
-docker==6.1.3
+python-dateutil==2.9.0
+numpy==1.26.4
+pyarrow==16.1.0
+pandas==2.2.0
+PyYAML==5.3.1
+docker-compose==1.29.2
 awscli==1.27.36
-requests==2.31.0
+docker==6.1.3
+requests==2.31.0

sink-connector/tests/integration/tests/primary_keys.py

@@ -47,6 +47,7 @@ def check_different_primary_keys(
             clickhouse_table=clickhouse_table,
             statement="id, Name",
             with_final=True,
+            order_by="id",
         )
 
 

sink-connector/tests/integration/tests/steps/sql.py

@@ -244,6 +244,7 @@ def select(
     with_optimize=False,
     sign_column="_sign",
     timeout=100,
+    order_by=None
 ):
     """SELECT with an option to either with FINAL or loop SELECT + OPTIMIZE TABLE default simple 'SELECT'
     :param insert: expected insert data if None compare with MySQL table
@@ -254,6 +255,11 @@ def select(
     :param with_optimize: loop 'OPTIMIZE TABLE' + 'SELECT'
     :param timeout: retry timeout
     """
+    if order_by is None:
+        order_by = ""
+    else:
+        order_by = f" ORDER BY {order_by}"
+
     if node is None:
         node = self.context.cluster.node("clickhouse")
     if table_name is None:
@@ -275,7 +281,7 @@ def select(
             timeout=timeout,
             delay=10,
         )(
-            f"SELECT {statement} FROM test.{table_name} FINAL  where {sign_column} !=-1 FORMAT CSV",
+            f"SELECT {statement} FROM test.{table_name} FINAL where {sign_column} !=-1{order_by} FORMAT CSV",
             message=f"{manual_output}",
         )
     elif with_optimize:
@@ -284,7 +290,7 @@ def select(
                 node.query(f"OPTIMIZE TABLE test.{table_name} FINAL DEDUPLICATE")
 
                 node.query(
-                    f"SELECT {statement} FROM test.{table_name} where {sign_column} !=-1 FORMAT CSV",
+                    f"SELECT {statement} FROM test.{table_name} where {sign_column} !=-1{order_by} FORMAT CSV",
                     message=f"{manual_output}",
                 )
 
@@ -294,7 +300,7 @@ def select(
             timeout=timeout,
             delay=10,
         )(
-            f"SELECT {statement} FROM test.{table_name} where {sign_column} !=-1 FORMAT CSV",
+            f"SELECT {statement} FROM test.{table_name} where {sign_column} !=-1{order_by} FORMAT CSV",
             message=f"{manual_output}",
         )
 
@@ -309,6 +315,7 @@ def complex_check_creation_and_select(
     manual_output=None,
     with_final=False,
     with_optimize=False,
+    order_by=None,
 ):
     """
     Check for table creation on all clickhouse nodes where it is expected and select data consistency with MySql
@@ -355,6 +362,7 @@ def complex_check_creation_and_select(
             with_final=with_final,
             with_optimize=with_optimize,
             timeout=timeout,
+            order_by=order_by,
         )
         if clickhouse_table[1].startswith("Replicated"):
             with Then(
@@ -369,6 +377,7 @@ def complex_check_creation_and_select(
                     with_final=with_final,
                     with_optimize=with_optimize,
                     timeout=timeout,
+                    order_by=order_by,
                 )