connectors-ci: write pytest logs to host (airbytehq#27042)

* connectors-ci: write pytest logs to host

* do no pass log as output artifact as they are already available as stdout or stderr

* only write acceptance test logs when local

.gitignore

@@ -15,6 +15,7 @@ static_checker_reports/
 
 # Logs
 acceptance_tests_logs/
+airbyte_ci_logs/
 
 # Secrets
 secrets

tools/ci_connector_ops/ci_connector_ops/pipelines/__init__.py

@@ -3,3 +3,8 @@
 #
 
 """The pipelines package."""
+import logging
+
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.WARNING)

tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py

@@ -15,8 +15,9 @@
 
 import anyio
 import asyncer
+from anyio import Path
 from ci_connector_ops.pipelines.consts import PYPROJECT_TOML_FILE_PATH
-from ci_connector_ops.pipelines.utils import check_path_in_workdir, with_exit_code, with_stderr, with_stdout
+from ci_connector_ops.pipelines.utils import check_path_in_workdir, slugify, with_exit_code, with_stderr, with_stdout
 from ci_connector_ops.utils import console
 from dagger import Container, QueryError
 from rich.console import Group
@@ -169,6 +170,13 @@ async def get_step_result(self, container: Container) -> StepResult:
 class PytestStep(Step, ABC):
     """An abstract class to run pytest tests and evaluate success or failure according to pytest logs."""
 
+    async def write_log_file(self, logs) -> str:
+        """Return the path to the pytest log file."""
+        log_directory = Path(f"{self.context.connector.code_directory}/airbyte_ci_logs")
+        await log_directory.mkdir(exist_ok=True)
+        await Path(f"{log_directory}/{slugify(self.title).replace('-', '_')}.log").write_text(logs)
+        self.context.logger.info(f"Pytest logs written to {log_directory}/{slugify(self.title)}.log")
+
     # TODO this is not very robust if pytest crashes and does not outputs its expected last log line.
     def pytest_logs_to_step_result(self, logs: str) -> StepResult:
         """Parse pytest log and infer failure, success or skipping.
@@ -214,7 +222,10 @@ async def _run_tests_in_directory(self, connector_under_test: Container, test_di
                     test_config,
                 ]
             )
-            return self.pytest_logs_to_step_result(await tester.stdout())
+            logs = await tester.stdout()
+            if self.context.is_local:
+                await self.write_log_file(logs)
+            return self.pytest_logs_to_step_result(logs)
 
         else:
             return StepResult(self, StepStatus.SKIPPED)

tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py

@@ -52,6 +52,7 @@ class PipelineContext:
         + glob("**/.eggs", recursive=True)
         + glob("**/.mypy_cache", recursive=True)
         + glob("**/.DS_Store", recursive=True)
+        + glob("**/airbyte_ci_logs", recursive=True)
     )
 
     def __init__(

tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py

@@ -193,5 +193,7 @@ async def _run(self, connector_under_test_image_tar: Optional[File]) -> StepResu
                 if file_path.startswith("updated_configurations"):
                     self.context.updated_secrets_dir = secret_dir
                     break
-
-        return self.pytest_logs_to_step_result(soon_cat_container_stdout.value)
+        logs = soon_cat_container_stdout.value
+        if self.context.is_local:
+            await self.write_log_file(logs)
+        return self.pytest_logs_to_step_result(logs)