Merge pull request #633 from Altinity/fix_broken_ci_cd

Fix broken CI/CD for TestFlows
Altinity · Jun 10, 2024 · 9bad7fa · 9bad7fa
2 parents 07a5a41 + d456bb0
commit 9bad7fa
Show file tree

Hide file tree

Showing 16 changed files with 65 additions and 29 deletions.
diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml
@@ -48,6 +48,18 @@ jobs:
         working-directory: sink-connector/tests/integration
         run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
 
+      - name: Create a virtual environment
+        run: |
+          echo "Install Python modules..."
+          sudo apt-get clean
+          sudo apt-get update
+          sudo apt-get install -y python3.12-venv
+          
+          echo "Create and activate Python virtual environment..."
+          python3 -m venv venv
+          source venv/bin/activate
+          echo PATH=$PATH >> $GITHUB_ENV
+
       - name: Install all dependencies
         working-directory: sink-connector/tests/integration
         run: pip3 install -r requirements.txt

diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml
@@ -64,6 +64,18 @@ jobs:
         working-directory: sink-connector/tests/integration
         run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
 
+      - name: Create a virtual environment
+        run: |
+          echo "Install Python modules..."
+          sudo apt-get clean
+          sudo apt-get update
+          sudo apt-get install -y python3.12-venv
+          
+          echo "Create and activate Python virtual environment..."
+          python3 -m venv venv
+          source venv/bin/activate
+          echo PATH=$PATH >> $GITHUB_ENV
+
       - name: Install all dependencies
         working-directory: sink-connector-lightweight/tests/integration
         run: pip3 install -r requirements.txt

diff --git a/sink-connector-lightweight/tests/integration/env/auto/docker-compose.yml b/sink-connector-lightweight/tests/integration/env/auto/docker-compose.yml
@@ -3,7 +3,7 @@ version: "2.3"
 
 services:
   mysql-master:
-    image: docker.io/bitnami/mysql:8.0
+    image: docker.io/bitnami/mysql:8.0.36
     restart: "no"
     expose:
       - "3306"

diff --git a/sink-connector-lightweight/tests/integration/requirements.txt b/sink-connector-lightweight/tests/integration/requirements.txt
@@ -1,4 +1,10 @@
-docker-compose==1.29.2
 testflows==2.1.5
+python-dateutil==2.9.0
+numpy==1.26.4
+pyarrow==16.1.0
+pandas==2.2.0
+PyYAML==5.3.1
+docker-compose==1.29.2
 awscli==1.27.36
-docker==6.1.3
+docker==6.1.3
+requests==2.31.0
diff --git a/sink-connector-lightweight/tests/integration/tests/calculated_columns.py b/sink-connector-lightweight/tests/integration/tests/calculated_columns.py
@@ -34,7 +34,7 @@ def string_concatenation(self):
         f"I create a {table_name} table with calculated column with string concatenation"
     ):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"first_name VARCHAR(50) NOT NULL,last_name VARCHAR(50) NOT NULL,fullname varchar(101) "
             f"GENERATED ALWAYS AS (CONCAT(first_name,' ',last_name)),email VARCHAR(100) NOT NULL",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
@@ -61,7 +61,7 @@ def basic_arithmetic_operations(self):
 
     with Given(f"I create a {table_name} table with calculated column"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"a INT, b INT, sum_col INT AS (a + b), diff_col INT AS (a - b), prod_col INT AS (a * b), div_col DOUBLE AS (a / b)",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )
@@ -88,7 +88,7 @@ def complex_expressions(self):
 
     with Given(f"I create a {table_name} table with calculated column"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"base_salary DECIMAL(10,2), bonus_rate DECIMAL(5,2), total_compensation DECIMAL(12,2) AS (base_salary + (base_salary * bonus_rate / 100))",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )
@@ -117,7 +117,7 @@ def nested(self):
 
     with Given(f"I create a {table_name} table with calculated column"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"a INT, b INT, c INT AS (a + b), d INT AS (c * 2)",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )

diff --git a/sink-connector-lightweight/tests/integration/tests/datatypes.py b/sink-connector-lightweight/tests/integration/tests/datatypes.py
@@ -36,7 +36,7 @@ def create_table_with_datetime_column(self, table_name, data, precision):
 
     with By(f"creating a {table_name} table with datetime column"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"date DATETIME({precision})",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )

diff --git a/sink-connector-lightweight/tests/integration/tests/is_deleted.py b/sink-connector-lightweight/tests/integration/tests/is_deleted.py
@@ -19,7 +19,7 @@ def create_table_with_is_deleted(
         f"creating a {table_name} table with is_deleted column and {datatype} datatype"
     ):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"col1 varchar(255), col2 int, {column} {datatype}",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )

diff --git a/sink-connector-lightweight/tests/integration/tests/multiple_databases.py b/sink-connector-lightweight/tests/integration/tests/multiple_databases.py
@@ -99,7 +99,7 @@ def create_table_and_insert_values(
 
     with By("creating a sample table in MySQL"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"col1 varchar(255), col2 int",
             database_name=database_name,
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],

diff --git a/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py b/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py
@@ -16,7 +16,7 @@ def retry_on_fail(self):
 
     with When("I creat a table in MySQL"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"retry VARCHAR(16)",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )

diff --git a/sink-connector-lightweight/tests/integration/tests/schema_only.py b/sink-connector-lightweight/tests/integration/tests/schema_only.py
@@ -15,7 +15,7 @@ def create_table_structure(self, table_name):
 
     with By(f"creating a {table_name} table"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"col1 varchar(255), col2 int",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )

diff --git a/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py b/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py
@@ -47,7 +47,7 @@ def create_and_validate_table(self, table_name):
         "creating a table in MySQL and checking that it was also created in ClickHouse"
     ):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns=f"col1 varchar(255), col2 int",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )

diff --git a/sink-connector-lightweight/tests/integration/tests/steps/alter.py b/sink-connector-lightweight/tests/integration/tests/steps/alter.py
@@ -20,7 +20,7 @@ def add_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type};"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type};"
     )
 
 
@@ -41,7 +41,7 @@ def rename_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` RENAME COLUMN {column_name} to {new_column_name};"
+        rf"ALTER TABLE {database}.\`{table_name}\` RENAME COLUMN {column_name} to {new_column_name};"
     )
 
 
@@ -63,7 +63,7 @@ def change_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` CHANGE COLUMN {column_name} {new_column_name} {new_column_type};"
+        rf"ALTER TABLE {database}.\`{table_name}\` CHANGE COLUMN {column_name} {new_column_name} {new_column_type};"
     )
 
 
@@ -84,7 +84,7 @@ def modify_column(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` MODIFY COLUMN {column_name} {new_column_type};"
+        rf"ALTER TABLE {database}.\`{table_name}\` MODIFY COLUMN {column_name} {new_column_type};"
     )
 
 
@@ -97,7 +97,7 @@ def drop_column(self, table_name, column_name="new_col", node=None, database=Non
     if node is None:
         node = self.context.cluster.node("mysql-master")
 
-    node.query(f"ALTER TABLE {database}.\`{table_name}\` DROP COLUMN {column_name};")
+    node.query(rf"ALTER TABLE {database}.\`{table_name}\` DROP COLUMN {column_name};")
 
 
 @TestStep(When)
@@ -149,7 +149,7 @@ def add_column_null_not_null(
 
     null_not_null = "NOT NULL" if not is_null else "NULL"
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} {null_not_null};"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} {null_not_null};"
     )
 
 
@@ -171,7 +171,7 @@ def add_column_default(
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} DEFAULT {default_value};"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} DEFAULT {default_value};"
     )
 
 
@@ -185,5 +185,5 @@ def add_primary_key(self, table_name, column_name, node=None, database=None):
         node = self.context.cluster.node("mysql-master")
 
     node.query(
-        f"ALTER TABLE {database}.\`{table_name}\` ADD PRIMARY KEY ({column_name});"
+        rf"ALTER TABLE {database}.\`{table_name}\` ADD PRIMARY KEY ({column_name});"
     )
diff --git a/sink-connector-lightweight/tests/integration/tests/steps/mysql.py b/sink-connector-lightweight/tests/integration/tests/steps/mysql.py
@@ -350,7 +350,7 @@ def insert(self, table_name, values, node=None, database_name=None):
         node = self.context.cluster.node("mysql-master")
 
     with When("I insert data into MySQL table"):
-        node.query(f"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});")
+        node.query(rf"INSERT INTO {database_name}.\`{table_name}\` VALUES ({values});")
 
 
 @TestStep(Given)

diff --git a/sink-connector-lightweight/tests/integration/tests/table_names.py b/sink-connector-lightweight/tests/integration/tests/table_names.py
@@ -52,25 +52,25 @@ def check_table_names(self, table_name):
 
     with Given(f"I create the {table_name} table"):
         create_mysql_to_clickhouse_replicated_table(
-            name=f"\`{table_name}\`",
+            name=rf"\`{table_name}\`",
             mysql_columns="x INT",
             clickhouse_columns="x Int32",
             clickhouse_table_engine=self.context.clickhouse_table_engines[0],
         )
 
     with And("I insert data into the table"):
-        mysql_node.query(f"INSERT INTO \`{table_name}\` VALUES (1, 1);")
+        mysql_node.query(rf"INSERT INTO \`{table_name}\` VALUES (1, 1);")
 
     with Then(f"I check that the {table_name} was created in the ClickHouse side"):
         for retry in retries(timeout=40, delay=1):
             with retry:
-                clickhouse_node.query(f"EXISTS test.\`{table_name}\`", message="1")
+                clickhouse_node.query(rf"EXISTS test.\`{table_name}\`", message="1")
 
     with And("I check that the data was inserted correctly into the ClickHouse table"):
         for retry in retries(timeout=40, delay=1):
             with retry:
                 clickhouse_data = clickhouse_node.query(
-                    f"SELECT id,x FROM test.\`{table_name}\` FORMAT CSV"
+                    rf"SELECT id,x FROM test.\`{table_name}\` FORMAT CSV"
                 )
                 assert clickhouse_data.output.strip() == "1,1", error()
 

diff --git a/sink-connector/tests/integration/env/docker-compose.yml b/sink-connector/tests/integration/env/docker-compose.yml
@@ -4,7 +4,7 @@ version: "2.3"
 services:
   mysql-master:
     container_name: mysql-master
-    image: docker.io/bitnami/mysql:8.0
+    image: docker.io/bitnami/mysql:8.0.36
     restart: "no"
     expose:
       - "3306"

diff --git a/sink-connector/tests/integration/requirements.txt b/sink-connector/tests/integration/requirements.txt
@@ -1,4 +1,10 @@
-docker-compose==1.29.2
 testflows==2.1.5
-docker==6.1.3
+python-dateutil==2.9.0
+numpy==1.26.4
+pyarrow==16.1.0
+pandas==2.2.0
+PyYAML==5.3.1
+docker-compose==1.29.2
 awscli==1.27.36
+docker==6.1.3
+requests==2.31.0