Stabilize database initialization in ydb/library/yql/providers/generic/connector/tests/datasource/ydb (#3844)

vitalyisaev2 · marsaly79 · commit 81c4618e1a6f · 2024-04-18T13:39:17.000+03:00
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/01_basic.sh b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/01_basic.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
-/ydb -p tests-ydb-client yql -s '
+set -ex
+
+/ydb -p ${PROFILE} yql -s '
     CREATE TABLE column_selection_A_b_C_d_E_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1));
     COMMIT;
     INSERT INTO column_selection_A_b_C_d_E_NATIVE (COL1, col2) VALUES
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/init_ydb b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/init_ydb
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+export PROFILE=tests-ydb-client-$(LC_ALL=C tr -dc A-Za-z0-9 </dev/urandom | head -c 8)
+
+# Wait until server is up
+timeout 30s bash -c 'until /ydb --endpoint grpc://fq-tests-ydb-ydb:2136 --database /local scheme ls; do sleep 3; done'
+
+# Run initialization scripts
+/ydb config profile create ${PROFILE} --endpoint grpc://fq-tests-ydb-ydb:2136 --database /local
+/bin/bash ./01_basic.sh
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py
@@ -228,13 +228,25 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
 
         # For debug add option --trace-opt to args
         cmd = f'{self.dqrun_path} -s -p {script_path} --fs-cfg={fs_conf_path} --gateways-cfg={gateways_conf_path} --result-file={result_path} --format="binary" -v 7'
-        out = subprocess.run(cmd, shell=True, capture_output=True)
 
+        output = None
         data_out = None
         data_out_with_types = None
         schema = None
+        returncode = 0
 
-        if out.returncode == 0:
+        try:
+            output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, timeout=60)
+        except subprocess.CalledProcessError as e:
+            LOGGER.error(
+                'Execution failed:\n\nSTDOUT: %s\n\nSTDERR: %s\n\n',
+                e.stdout.decode('utf-8') if e.stdout else None,
+                e.stderr.decode('utf-8') if e.stderr else None,
+            )
+
+            output = e.stdout
+            returncode = e.returncode
+        else:
             LOGGER.info('Execution succeeded: ')
             # Parse output
             with open(result_path, 'r') as f:
@@ -250,24 +262,15 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
 
             artifacts.dump_yson(data_out, test_name, "data_out.yson")
             artifacts.dump_str(data_out_with_types, test_name, "data_out_with_types.yson")
-        else:
-            LOGGER.error(
-                'Execution failed:\n\nSTDOUT: %s\n\nSTDERR: %s\n\n',
-                out.stdout.decode('utf-8'),
-                out.stderr.decode('utf-8'),
-            )
-
-        with open(artifacts.make_path(test_name, "dqrun.err"), "w") as f:
-            f.write(out.stderr.decode('utf-8'))
 
-        with open(artifacts.make_path(test_name, "dqrun.out"), "w") as f:
-            f.write(out.stdout.decode('utf-8'))
+        finally:
+            with open(artifacts.make_path(test_name, "kqprun.out"), "w") as f:
+                f.write(output.decode('utf-8'))
 
         return Result(
             data_out=data_out,
             data_out_with_types=data_out_with_types,
             schema=schema,
-            stdout=out.stdout.decode('utf-8'),
-            stderr=out.stderr.decode('utf-8'),
-            returncode=out.returncode,
+            output=output.decode('utf-8') if output else None,
+            returncode=returncode,
         )
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py
@@ -192,13 +192,25 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
 
         # For debug add option --trace-opt to args
         cmd = f'{self.kqprun_path} -s {scheme_path} -p {script_path} --app-config={app_conf_path} --result-file={result_path} --result-format=full'
-        out = subprocess.run(cmd, shell=True, capture_output=True)
 
+        output = None
         data_out = None
         data_out_with_types = None
         schema = None
+        returncode = 0
 
-        if out.returncode == 0:
+        try:
+            output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, timeout=60)
+        except subprocess.CalledProcessError as e:
+            LOGGER.error(
+                'Execution failed:\n\nSTDOUT: %s\n\nSTDERR: %s\n\n',
+                e.stdout.decode('utf-8') if e.stdout else None,
+                e.stderr.decode('utf-8') if e.stderr else None,
+            )
+
+            output = e.stdout
+            returncode = e.returncode
+        else:
             # Parse output
             with open(result_path, 'r') as f:
                 result = json.loads(f.read().encode('ascii'), strict=False)
@@ -225,24 +237,14 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
             artifacts.dump_json(data_out, test_name, "data_out.yson")
             artifacts.dump_str(data_out_with_types, test_name, "data_out_with_types.yson")
 
-        else:
-            LOGGER.error(
-                'Execution failed:\n\nSTDOUT: %s\n\nSTDERR: %s\n\n',
-                out.stdout.decode('utf-8'),
-                out.stderr.decode('utf-8'),
-            )
-
-        with open(artifacts.make_path(test_name, "kqprun.err"), "w") as f:
-            f.write(out.stderr.decode('utf-8'))
-
-        with open(artifacts.make_path(test_name, "kqprun.out"), "w") as f:
-            f.write(out.stdout.decode('utf-8'))
+        finally:
+            with open(artifacts.make_path(test_name, "kqprun.out"), "w") as f:
+                f.write(output.decode('utf-8'))
 
         return Result(
             data_out=data_out,
             data_out_with_types=data_out_with_types,
             schema=schema,
-            stdout=out.stdout.decode('utf-8'),
-            stderr=out.stderr.decode('utf-8'),
-            returncode=out.returncode,
+            output=output.decode('utf-8') if output else None,
+            returncode=returncode,
         )
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/result.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/result.py
@@ -10,6 +10,5 @@ class Result:
     data_out: Optional[yson.yson_types.YsonList]
     data_out_with_types: Optional[List]
     schema: Optional[Schema]
-    stdout: str
-    stderr: str
+    output: str
     returncode: int
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py
@@ -101,7 +101,7 @@ def select_positive(
         generic_settings=test_case.generic_settings,
     )
 
-    assert result.returncode == 0, result.stderr
+    assert result.returncode == 0, result.output
 
     assert data_outs_equal(test_case.data_out, result.data_out_with_types), (
         test_case.data_out,
@@ -128,7 +128,7 @@ def select_missing_database(
         generic_settings=test_case.generic_settings,
     )
 
-    assert test_case.database.missing_database_msg() in result.stderr, result.stderr
+    assert test_case.database.missing_database_msg() in result.output, result.output
 
 
 def select_missing_table(
@@ -153,4 +153,4 @@ def select_missing_table(
         generic_settings=test_case.generic_settings,
     )
 
-    assert test_case.database.missing_table_msg() in result.stderr, result.stderr
+    assert test_case.database.missing_table_msg() in result.output, result.output
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py
@@ -118,7 +118,7 @@ def select_positive(
         generic_settings=test_case.generic_settings,
     )
 
-    assert result.returncode == 0, result.stderr
+    assert result.returncode == 0, result.output
 
     assert data_outs_equal(test_case.data_out, result.data_out_with_types), (
         test_case.data_out,
@@ -146,7 +146,7 @@ def select_missing_database(
         generic_settings=test_case.generic_settings,
     )
 
-    assert test_case.database.missing_database_msg() in result.stderr, result.stderr
+    assert test_case.database.missing_database_msg() in result.output, result.output
 
 
 def select_missing_table(
@@ -182,7 +182,7 @@ def select_missing_table(
         generic_settings=test_case.generic_settings,
     )
 
-    assert test_case.database.missing_table_msg() in result.stderr, result.stderr
+    assert test_case.database.missing_table_msg() in result.output, result.output
 
 
 def select_pg_schema(
@@ -213,7 +213,7 @@ def select_pg_schema(
         generic_settings=test_case.generic_settings,
     )
 
-    assert result.returncode == 0, result.stderr
+    assert result.returncode == 0, result.output
 
     assert data_outs_equal(test_case.data_out, result.data_out_with_types), (
         test_case.data_out,

Original file line number	Diff line number	Diff line change
`@@ -101,7 +101,7 @@ def select_positive(`
`101`	`101`	`generic_settings=test_case.generic_settings,`
`102`	`102`	`)`
`103`	`103`
`104`		`- assert result.returncode == 0, result.stderr`
	`104`	`+ assert result.returncode == 0, result.output`
`105`	`105`
`106`	`106`	`assert data_outs_equal(test_case.data_out, result.data_out_with_types), (`
`107`	`107`	`test_case.data_out,`
`@@ -128,7 +128,7 @@ def select_missing_database(`
`128`	`128`	`generic_settings=test_case.generic_settings,`
`129`	`129`	`)`
`130`	`130`
`131`		`- assert test_case.database.missing_database_msg() in result.stderr, result.stderr`
	`131`	`+ assert test_case.database.missing_database_msg() in result.output, result.output`
`132`	`132`
`133`	`133`
`134`	`134`	`def select_missing_table(`
`@@ -153,4 +153,4 @@ def select_missing_table(`
`153`	`153`	`generic_settings=test_case.generic_settings,`
`154`	`154`	`)`
`155`	`155`
`156`		`- assert test_case.database.missing_table_msg() in result.stderr, result.stderr`
	`156`	`+ assert test_case.database.missing_table_msg() in result.output, result.output`
Original file line number	Diff line number	Diff line change
`@@ -118,7 +118,7 @@ def select_positive(`
`118`	`118`	`generic_settings=test_case.generic_settings,`
`119`	`119`	`)`
`120`	`120`
`121`		`- assert result.returncode == 0, result.stderr`
	`121`	`+ assert result.returncode == 0, result.output`
`122`	`122`
`123`	`123`	`assert data_outs_equal(test_case.data_out, result.data_out_with_types), (`
`124`	`124`	`test_case.data_out,`
`@@ -146,7 +146,7 @@ def select_missing_database(`
`146`	`146`	`generic_settings=test_case.generic_settings,`
`147`	`147`	`)`
`148`	`148`
`149`		`- assert test_case.database.missing_database_msg() in result.stderr, result.stderr`
	`149`	`+ assert test_case.database.missing_database_msg() in result.output, result.output`
`150`	`150`
`151`	`151`
`152`	`152`	`def select_missing_table(`
`@@ -182,7 +182,7 @@ def select_missing_table(`
`182`	`182`	`generic_settings=test_case.generic_settings,`
`183`	`183`	`)`
`184`	`184`
`185`		`- assert test_case.database.missing_table_msg() in result.stderr, result.stderr`
	`185`	`+ assert test_case.database.missing_table_msg() in result.output, result.output`
`186`	`186`
`187`	`187`
`188`	`188`	`def select_pg_schema(`
`@@ -213,7 +213,7 @@ def select_pg_schema(`
`213`	`213`	`generic_settings=test_case.generic_settings,`
`214`	`214`	`)`
`215`	`215`
`216`		`- assert result.returncode == 0, result.stderr`
	`216`	`+ assert result.returncode == 0, result.output`
`217`	`217`
`218`	`218`	`assert data_outs_equal(test_case.data_out, result.data_out_with_types), (`
`219`	`219`	`test_case.data_out,`