Trying to debug kqprun pt. 2

Author: vitalyisaev2

ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py

@@ -228,13 +228,25 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
 
         # For debug add option --trace-opt to args
         cmd = f'{self.dqrun_path} -s -p {script_path} --fs-cfg={fs_conf_path} --gateways-cfg={gateways_conf_path} --result-file={result_path} --format="binary" -v 7'
-        out = subprocess.run(cmd, shell=True, capture_output=True)
 
+        output = None
         data_out = None
         data_out_with_types = None
         schema = None
+        returncode = 0
 
-        if out.returncode == 0:
+        try:
+            output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, timeout=60)
+        except subprocess.CalledProcessError as e:
+            LOGGER.error(
+                'Execution failed:\n\nSTDOUT: %s\n\nSTDERR: %s\n\n',
+                e.stdout.decode('utf-8') if e.stdout else None,
+                e.stderr.decode('utf-8') if e.stderr else None,
+            )
+
+            output = e.stdout
+            returncode = e.returncode
+        else:
             LOGGER.info('Execution succeeded: ')
             # Parse output
             with open(result_path, 'r') as f:
@@ -250,24 +262,15 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
 
             artifacts.dump_yson(data_out, test_name, "data_out.yson")
             artifacts.dump_str(data_out_with_types, test_name, "data_out_with_types.yson")
-        else:
-            LOGGER.error(
-                'Execution failed:\n\nSTDOUT: %s\n\nSTDERR: %s\n\n',
-                out.stdout.decode('utf-8'),
-                out.stderr.decode('utf-8'),
-            )
-
-        with open(artifacts.make_path(test_name, "dqrun.err"), "w") as f:
-            f.write(out.stderr.decode('utf-8'))
 
-        with open(artifacts.make_path(test_name, "dqrun.out"), "w") as f:
-            f.write(out.stdout.decode('utf-8'))
+        finally:
+            with open(artifacts.make_path(test_name, "kqprun.out"), "w") as f:
+                f.write(output.decode('utf-8'))
 
         return Result(
             data_out=data_out,
             data_out_with_types=data_out_with_types,
             schema=schema,
-            stdout=out.stdout.decode('utf-8'),
-            stderr=out.stderr.decode('utf-8'),
-            returncode=out.returncode,
+            output=output.decode('utf-8') if output else None,
+            returncode=returncode,
         )

ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py

@@ -193,24 +193,22 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
         # For debug add option --trace-opt to args
         cmd = f'{self.kqprun_path} -s {scheme_path} -p {script_path} --app-config={app_conf_path} --result-file={result_path} --result-format=full'
 
-        stdout = None
-        stderr = None 
+        output = None
         data_out = None
         data_out_with_types = None
         schema = None
         returncode = 0
 
         try:
-            stdout = subprocess.check_output(cmd, shell=True, timeout=60)
+            output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, timeout=60)
         except subprocess.CalledProcessError as e:
             LOGGER.error(
                 'Execution failed:\n\nSTDOUT: %s\n\nSTDERR: %s\n\n',
                 e.stdout.decode('utf-8') if e.stdout else None,
                 e.stderr.decode('utf-8') if e.stderr else None,
             )
 
-            stdout = e.stdout
-            stderr = e.stderr
+            output = e.stdout
             returncode = e.returncode
         else:
             # Parse output
@@ -240,18 +238,13 @@ def run(self, test_name: str, script: str, generic_settings: GenericSettings) ->
             artifacts.dump_str(data_out_with_types, test_name, "data_out_with_types.yson")
 
         finally:
-            if stderr:
-                with open(artifacts.make_path(test_name, "kqprun.err"), "w") as f:
-                    f.write(stderr.decode('utf-8'))
-
             with open(artifacts.make_path(test_name, "kqprun.out"), "w") as f:
-                f.write(stdout.decode('utf-8'))
+                f.write(output.decode('utf-8'))
 
         return Result(
             data_out=data_out,
             data_out_with_types=data_out_with_types,
             schema=schema,
-            stdout=stdout.decode('utf-8') if stdout else None,
-            stderr=stderr.decode('utf-8') if stderr else None,
+            output=output.decode('utf-8') if output else None,
             returncode=returncode,
         )

ydb/library/yql/providers/generic/connector/tests/utils/run/result.py

@@ -10,6 +10,5 @@ class Result:
     data_out: Optional[yson.yson_types.YsonList]
     data_out_with_types: Optional[List]
     schema: Optional[Schema]
-    stdout: str
-    stderr: str
+    output: str
     returncode: int

ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py

@@ -101,7 +101,7 @@ def select_positive(
         generic_settings=test_case.generic_settings,
     )
 
-    assert result.returncode == 0, result.stderr
+    assert result.returncode == 0, result.output
 
     assert data_outs_equal(test_case.data_out, result.data_out_with_types), (
         test_case.data_out,
@@ -128,7 +128,7 @@ def select_missing_database(
         generic_settings=test_case.generic_settings,
     )
 
-    assert test_case.database.missing_database_msg() in result.stderr, result.stderr
+    assert test_case.database.missing_database_msg() in result.output, result.output
 
 
 def select_missing_table(
@@ -153,4 +153,4 @@ def select_missing_table(
         generic_settings=test_case.generic_settings,
     )
 
-    assert test_case.database.missing_table_msg() in result.stderr, result.stderr
+    assert test_case.database.missing_table_msg() in result.output, result.output