From 771bc263822397f01b48afd3f70680ed00da9f5b Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Tue, 10 Oct 2023 13:15:02 -0700 Subject: [PATCH] gradle: search for python3.11 binary for AL2023 support (#31227) --- airbyte-ci/connectors/pipelines/README.md | 1 + airbyte-ci/connectors/pipelines/pipelines/gradle.py | 2 +- .../pipelines/pipelines/tests/java_connectors.py | 2 +- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- build.gradle | 12 ++++++++++++ buildSrc/src/main/groovy/airbyte-python.gradle | 12 ++++++++++++ 6 files changed, 28 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 77b18e5f0792..19e90598569d 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -395,6 +395,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 1.5.1 | [#31227](https://github.com/airbytehq/airbyte/pull/31227) | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'. | | 1.5.0 | [#30456](https://github.com/airbytehq/airbyte/pull/30456) | Start building Python connectors using our base images. | | 1.4.6 | [ #31087](https://github.com/airbytehq/airbyte/pull/31087) | Throw error if airbyte-ci tools is out of date | | 1.4.5 | [#31133](https://github.com/airbytehq/airbyte/pull/31133) | Fix bug when building containers using `with_integration_base_java_and_normalization`. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/gradle.py index 362cc429593c..cf20a8079417 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/gradle.py @@ -90,7 +90,7 @@ async def _run(self) -> StepResult: "findutils", # gradle requires xargs, which is shipped in findutils. "jq", # required by :airbyte-connector-test-harnesses:acceptance-test-harness to inspect docker images. "npm", # required by :format. - "pip", # required by :format. + "python3.11-pip", # required by :format. "rsync", # required for gradle cache synchronization. ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py index 5d7c15211248..1859f3b5dd67 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py @@ -55,7 +55,7 @@ class UnitTests(GradleTask): """A step to run unit tests for Java connectors.""" title = "Java Connector Unit Tests" - gradle_task_name = "check" + gradle_task_name = "test" bind_to_docker_host = True diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 905a1ef0c6f7..b85169bdc9e2 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "1.5.0" +version = "1.5.1" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/build.gradle b/build.gradle index c2d521a76d9a..9d92d5d19eba 100644 --- a/build.gradle +++ b/build.gradle @@ -159,6 +159,18 @@ python { envPath = '.venv' minPythonVersion = '3.10' // should be 3.10 for local development + // Amazon Linux support. + // The airbyte-ci tool runs gradle tasks in AL2023-based containers. + // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. + // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. + try { + if ("python3.11 --version".execute().waitFor() == 0) { + // python3.11 definitely exists at this point, use it instead of 'python3'. + pythonBinary "python3.11" + } + } catch (IOException _) { + // Swallow exception if python3.11 is not installed. + } // Pyenv support. try { def pyenvRoot = "pyenv root".execute() diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index b0be7c885051..03bdb06b7b8b 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -80,6 +80,18 @@ class AirbytePythonPlugin implements Plugin { envPath = venvDirectoryName minPythonVersion '3.10' + // Amazon Linux support. + // The airbyte-ci tool runs gradle tasks in AL2023-based containers. + // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. + // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. + try { + if ("python3.11 --version".execute().waitFor() == 0) { + // python3.11 definitely exists at this point, use it instead of 'python3'. + pythonBinary "python3.11" + } + } catch (IOException _) { + // Swallow exception if python3.11 is not installed. + } // Pyenv support. try { def pyenvRoot = "pyenv root".execute()