connectors-ci: better modified connectors detection logic

.github/workflows/publish_connectors.yml

@@ -40,7 +40,7 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
           spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
-          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --modified publish --main-release"
+          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
 
       - name: Publish connectors [manual]
         id: publish-connectors

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -319,7 +319,7 @@ def __repr__(self) -> str:
 
     @functools.lru_cache(maxsize=2)
     def get_local_dependency_paths(self, with_test_dependencies: bool = True) -> Set[Path]:
-        dependencies_paths = [self.code_directory]
+        dependencies_paths = []
         if self.language == ConnectorLanguage.JAVA:
             dependencies_paths += get_all_gradle_dependencies(
                 self.code_directory / "build.gradle", with_test_dependencies=with_test_dependencies
@@ -351,9 +351,18 @@ def get_changed_connectors(
     return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files}
 
 
-def get_all_released_connectors() -> Set:
+def get_all_connectors_in_repo() -> Set[Connector]:
+    """Retrieve a set of all Connectors in the repo.
+    We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name.
+
+    Returns:
+        A set of Connectors.
+    """
+    repo = git.Repo(search_parent_directories=True)
+    repo_path = repo.working_tree_dir
+
     return {
         Connector(Path(metadata_file).parent.name)
-        for metadata_file in glob("airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
+        for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
         if SCAFFOLD_CONNECTOR_GLOB not in metadata_file
     }

airbyte-ci/connectors/connector_ops/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "connector_ops"
-version = "0.2.1"
+version = "0.2.3"
 description = "Packaged maintained by the connector operations team to perform CI for connectors"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/README.md

@@ -123,6 +123,8 @@ Available commands:
 | `--language`           | True     |               | Select connectors with a specific language: `python`, `low-code`, `java`. Can be used multiple times to select multiple languages.                                                                                                                                                                    |
 | `--modified`           | False    | False         | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation).                                                                                                                                                                           |
 | `--concurrency`        | False    | 5             | Control the number of connector pipelines that can run in parallel. Useful to speed up pipelines or control their resource usage.                                                                                                                                                                     |
+| `--metadata-change-only/--not-metadata-change-only`        | False    | `--not-metadata-change-only`             | Only run the pipeline on connectors with changes on their metadata.yaml file.                                                                                                                                                                     |
+
 
 ### <a id="connectors-list-command"></a>`connectors list` command
 Retrieve the list of connectors satisfying the provided filters.
@@ -378,8 +380,9 @@ This command runs the Python tests for a airbyte-ci poetry package.
 
 | Version | PR                                                        | Description                                                                                  |
 |---------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------|
-| 0.3.2   | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull.                                              |
-| 0.3.1   | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped                                |
+| 0.3.3   | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands.                           |
+| 0.3.2   | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull.                                                |
+| 0.3.1   | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped                                            |
 | 0.3.0   | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution                                |
 | 0.2.3   | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command                                    |
 | 0.2.2   | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting                                  |

airbyte-ci/connectors/pipelines/pipelines/bases.py

@@ -13,18 +13,18 @@
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import TYPE_CHECKING, Any, ClassVar, List, Optional
+from typing import TYPE_CHECKING, Any, ClassVar, List, Optional, Set
 
 import anyio
 import asyncer
 from anyio import Path
-from connector_ops.utils import console
+from connector_ops.utils import Connector, console
 from dagger import Container, DaggerError, QueryError
 from jinja2 import Environment, PackageLoader, select_autoescape
 from pipelines import sentry_utils
 from pipelines.actions import remote_storage
 from pipelines.consts import GCS_PUBLIC_DOMAIN, LOCAL_REPORTS_PATH_ROOT, PYPROJECT_TOML_FILE_PATH
-from pipelines.utils import check_path_in_workdir, format_duration, get_exec_result
+from pipelines.utils import METADATA_FILE_NAME, check_path_in_workdir, format_duration, get_exec_result
 from rich.console import Group
 from rich.panel import Panel
 from rich.style import Style
@@ -36,6 +36,15 @@
     from pipelines.contexts import PipelineContext
 
 
+@dataclass(frozen=True)
+class ConnectorWithModifiedFiles(Connector):
+    modified_files: Set[Path] = field(default_factory=list)
+
+    @property
+    def has_metadata_change(self) -> bool:
+        return any(path.name == METADATA_FILE_NAME for path in self.modified_files)
+
+
 class CIContext(str, Enum):
     """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"]."""
 

airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py

@@ -17,6 +17,7 @@
     get_modified_files_in_branch,
     get_modified_files_in_commit,
     get_modified_files_in_pull_request,
+    transform_strs_to_paths,
 )
 
 from .groups.connectors import connectors
@@ -120,7 +121,9 @@ def airbyte_ci(
     else:
         ctx.obj["pull_request"] = None
 
-    ctx.obj["modified_files"] = get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"])
+    ctx.obj["modified_files"] = transform_strs_to_paths(
+        get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"])
+    )
 
     if not is_local:
         main_logger.info("Running airbyte-ci in CI mode.")
@@ -134,6 +137,7 @@ def airbyte_ci(
         main_logger.info(f"Pipeline Start Timestamp: {pipeline_start_timestamp}")
         main_logger.info(f"Modified Files: {ctx.obj['modified_files']}")
 
+
 airbyte_ci.add_command(connectors)
 airbyte_ci.add_command(metadata)
 airbyte_ci.add_command(tests)