Update unit tests.

Add changed module filter.

tools/ci_static_check_reports/ci_build_python_static_checkers_reports/main.py

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
+import argparse
 import json
 import os
 import sys
@@ -55,16 +56,16 @@
 }
 
 
-def build_static_checkers_reports(modules: list) -> int:
+def build_static_checkers_reports(modules: list, static_checker_reports_path: str) -> int:
     ctx = Context()
     toml_config_file = os.path.join(os.getcwd(), "pyproject.toml")
 
     for module_path in modules:
-        reports_path = f"{os.getcwd()}/static_checker_reports/{module_path}"
+        reports_path = f"{os.getcwd()}/{static_checker_reports_path}/{module_path}"
         if not os.path.exists(reports_path):
             os.makedirs(reports_path)
 
-        for checker in TASK_COMMANDS.keys():
+        for checker in TASK_COMMANDS:
             _run_task(
                 ctx,
                 f"{os.getcwd()}/{module_path}",
@@ -81,9 +82,18 @@ def build_static_checkers_reports(modules: list) -> int:
 
 
 def main() -> int:
-    changed_python_module_paths = [module["dir"] for module in json.loads(sys.argv[1]) if module["lang"] == "py"]
+    parser = argparse.ArgumentParser(description="Working with Python Static Report Builder.")
+    parser.add_argument("changed_modules", nargs="*")
+    parser.add_argument("--static-checker-reports-path", help="SonarQube host", required=False, type=str, default="static_checker_reports")
+
+    args = parser.parse_args()
+    changed_python_module_paths = [
+        module["dir"]
+        for module in json.loads(args.changed_modules[0])
+        if module["lang"] == "py" and os.path.exists(module["dir"]) and "setup.py" in os.listdir(module["dir"])
+    ]
     print("Changed python modules: ", changed_python_module_paths)
-    return build_static_checkers_reports(changed_python_module_paths)
+    return build_static_checkers_reports(changed_python_module_paths, static_checker_reports_path=args.static_checker_reports_path)
 
 
 if __name__ == "__main__":

tools/ci_static_check_reports/ci_detect_changed_modules/main.py

@@ -29,6 +29,11 @@ def find_base_path(path: str, modules: List[Dict[str, str]], unique_modules: Set
 
 
 def list_changed_modules(changed_files: List[str]) -> List[Dict[str, str]]:
+    """
+    changed_filed are the list of files which were modified in current branch.
+    E.g. changed_files = ["tools/ci_static_check_reports/__init__.py", "tools/ci_static_check_reports/setup.py", ...]
+    """
+
     modules: List[Dict[str, str]] = []
     unique_modules: set = set()
     for file_path in changed_files:

tools/ci_static_check_reports/unit_tests/test_build_static_checkers_reports.py

@@ -2,19 +2,41 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 import os
+import subprocess
 
-from ci_build_static_checkers_reports.main import build_static_checkers_reports
+import pytest
 
 
-def test_build_static_checkers_reports() -> None:
-    changed_module_path = "tools/ci_static_check_reports"
-    build_static_checkers_reports([changed_module_path])
-    static_checker_reports_path = "static_checker_reports/tools/ci_static_check_reports"
+@pytest.mark.parametrize(
+    "changed_module,should_build_reports",
+    [
+        ('[{"dir": "tools/ci_static_check_reports", "lang": "py"}]', True),
+        ('[{"dir": "airbyte-integrations/connectors/destination-bigquery", "lang": "java"}]', False),
+        ('[{"dir": "airbyte-integrations/connectors/not-existing-module", "lang": "other"}]', False),
+    ],
+)
+def test_build_static_checkers_reports(changed_module: str, should_build_reports: bool) -> None:
+    subprocess.call(["ci_build_python_checkers_reports", changed_module], shell=False)
+    static_checker_reports_path = f"static_checker_reports/{changed_module}"
 
-    assert os.path.exists(static_checker_reports_path)
-    assert os.path.exists(os.path.join(static_checker_reports_path, "black.txt"))
-    assert os.path.exists(os.path.join(static_checker_reports_path, "coverage.xml"))
-    assert os.path.exists(os.path.join(static_checker_reports_path, "flake.xml"))
-    assert os.path.exists(os.path.join(static_checker_reports_path, "isort.txt"))
-    assert os.path.exists(os.path.join(static_checker_reports_path, "cobertura.xml"))
-    assert os.path.exists(os.path.join(static_checker_reports_path, "pytest.xml"))
+    static_checker_reports_path_exists = os.path.exists(static_checker_reports_path)
+    black_exists = os.path.exists(os.path.join(static_checker_reports_path, "black.txt"))
+    coverage_exists = os.path.exists(os.path.join(static_checker_reports_path, "coverage.xml"))
+    flake_exists = os.path.exists(os.path.join(static_checker_reports_path, "flake.xml"))
+    isort_exists = os.path.exists(os.path.join(static_checker_reports_path, "isort.txt"))
+    cobertura_exists = os.path.exists(os.path.join(static_checker_reports_path, "cobertura.xml"))
+    pytest_exists = os.path.exists(os.path.join(static_checker_reports_path, "pytest.xml"))
+    report_paths_exist = [
+        static_checker_reports_path_exists,
+        black_exists,
+        coverage_exists,
+        flake_exists,
+        isort_exists,
+        cobertura_exists,
+        pytest_exists,
+    ]
+
+    if should_build_reports:
+        assert all(report_paths_exist)
+    else:
+        assert not all(report_paths_exist)

tools/ci_static_check_reports/unit_tests/test_detect_changed_modules.py

@@ -4,21 +4,20 @@
 from typing import List, Set
 
 import pytest
-
 from ci_detect_changed_modules.main import list_changed_modules
 
 
 @pytest.mark.parametrize(
     "changed_files,changed_modules",
     [
-        (["path/to/file1", "file2.txt", "path/to/file3.txt"], set()),
+        (["path/to/file1", "file2.txt", "path/to/file3.txt"], []),
         (
             [
                 "airbyte-cdk/python/airbyte_cdk/entrypoint.py",
                 "airbyte-cdk/python/airbyte_cdk/file1",
                 "airbyte-cdk/python/airbyte_cdk/file2.py",
             ],
-            {"airbyte-cdk/python"},
+            [{"dir": "airbyte-cdk/python", "lang": "py"}],
         ),
         (
             [
@@ -27,7 +26,29 @@
                 "airbyte-integrations/connectors/source-asana/source_asana/source.py",
                 "airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json",
             ],
-            {"airbyte-cdk/python", "airbyte-integrations/connectors/source-asana"},
+            [{"dir": "airbyte-cdk/python", "lang": "py"}, {"dir": "airbyte-integrations/connectors/source-asana", "lang": "py"}],
+        ),
+        (
+            [],
+            [],
+        ),
+        # TODO: update test after non-python modules are supported
+        (
+            [
+                "airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/"
+                "java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java"
+            ],
+            [],
+        ),
+        (
+            ["airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json"],
+            [],
+        ),
+        (
+            ["airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py"],
+            [
+                {"dir": "airbyte-integrations/connectors/destination-amazon-sqs", "lang": "py"},
+            ],
         ),
     ],
 )