Make latest botocore tests green (apache#41626)

The latest botocore tests are conflicting with a few requirements
and until apache-beam upcoming version is released we need to do
some manual exclusions. Those exclusions should make latest botocore
test green again.

Dockerfile.ci

@@ -1013,16 +1013,17 @@ function check_boto_upgrade() {
     echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
     echo
     # shellcheck disable=SC2086
-    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true
+    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true
     # We need to include few dependencies to pass pip check with other dependencies:
     #   * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider)
-    #   * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider)
+    #   * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider)
     #   * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider)
     #   * yandexcloud requirements for requests does not match those of apache.beam and latest botocore
     #   Both requests and yandexcloud exclusion above might be removed after
     #   https://github.com/apache/beam/issues/32080 is addressed
-    #   When you remove yandexcloud from the above list, also remove it from "test_example_dags.py"
-    #   in "tests/always".
+    #   This is already addressed and planned for 2.59.0 release.
+    #   When you remove yandexcloud and opensearch from the above list, you can also remove the
+    #   optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always".
     set -x
     # shellcheck disable=SC2086
     ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \

pyproject.toml

@@ -406,6 +406,9 @@ combine-as-imports = true
 "tests/providers/google/cloud/operators/vertex_ai/test_generative_model.py" = ["E402"]
 "tests/providers/google/cloud/triggers/test_vertex_ai.py" = ["E402"]
 "tests/providers/openai/hooks/test_openai.py" = ["E402"]
+"tests/providers/opensearch/conftest.py" = ["E402"]
+"tests/providers/opensearch/hooks/test_opensearch.py" = ["E402"]
+"tests/providers/opensearch/operators/test_opensearch.py" = ["E402"]
 "tests/providers/openai/operators/test_openai.py" = ["E402"]
 "tests/providers/qdrant/hooks/test_qdrant.py" = ["E402"]
 "tests/providers/qdrant/operators/test_qdrant.py" = ["E402"]

scripts/docker/entrypoint_ci.sh

@@ -250,16 +250,17 @@ function check_boto_upgrade() {
     echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
     echo
     # shellcheck disable=SC2086
-    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true
+    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true
     # We need to include few dependencies to pass pip check with other dependencies:
     #   * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider)
-    #   * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider)
+    #   * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider)
     #   * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider)
     #   * yandexcloud requirements for requests does not match those of apache.beam and latest botocore
     #   Both requests and yandexcloud exclusion above might be removed after
     #   https://github.com/apache/beam/issues/32080 is addressed
-    #   When you remove yandexcloud from the above list, also remove it from "test_example_dags.py"
-    #   in "tests/always".
+    #   This is already addressed and planned for 2.59.0 release.
+    #   When you remove yandexcloud and opensearch from the above list, you can also remove the
+    #   optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always".
     set -x
     # shellcheck disable=SC2086
     ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \

tests/always/test_example_dags.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import os
+import re
 import sys
 from glob import glob
 from importlib import metadata as importlib_metadata
@@ -38,8 +39,11 @@
     # Some examples or system tests may depend on additional packages
     # that are not included in certain CI checks.
     # The format of the dictionary is as follows:
-    # key: the prefix of the file to be excluded,
+    # key: the regexp matching the file to be excluded,
     # value: a dictionary containing package distributions with an optional version specifier, e.g., >=2.3.4
+    ".*example_bedrock_retrieve_and_generate.py": {"opensearch-py": None},
+    ".*example_opensearch.py": {"opensearch-py": None},
+    r".*example_yandexcloud.*\.py": {"yandexcloud": None},
 }
 IGNORE_AIRFLOW_PROVIDER_DEPRECATION_WARNING: tuple[str, ...] = (
     # Certain examples or system tests may trigger AirflowProviderDeprecationWarnings.
@@ -123,13 +127,6 @@ def example_not_excluded_dags(xfail_db_exception: bool = False):
         for prefix in PROVIDERS_PREFIXES
         for provider in suspended_providers_folders
     ]
-    temporary_excluded_upgrade_boto_providers_folders = [
-        AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix()
-        for prefix in PROVIDERS_PREFIXES
-        # TODO - remove me when https://github.com/apache/beam/issues/32080 is addressed
-        #        and we bring back yandex to be run in case of upgrade boto
-        for provider in ["yandex"]
-    ]
     current_python_excluded_providers_folders = [
         AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix()
         for prefix in PROVIDERS_PREFIXES
@@ -145,18 +142,13 @@ def example_not_excluded_dags(xfail_db_exception: bool = False):
             if candidate.startswith(tuple(suspended_providers_folders)):
                 param_marks.append(pytest.mark.skip(reason="Suspended provider"))
 
-            if os.environ.get("UPGRADE_BOTO", "false") == "true" and candidate.startswith(
-                tuple(temporary_excluded_upgrade_boto_providers_folders)
-            ):
-                param_marks.append(pytest.mark.skip(reason="Temporary excluded upgrade boto provider"))
-
             if candidate.startswith(tuple(current_python_excluded_providers_folders)):
                 param_marks.append(
                     pytest.mark.skip(reason=f"Not supported for Python {CURRENT_PYTHON_VERSION}")
                 )
 
             for optional, dependencies in OPTIONAL_PROVIDERS_DEPENDENCIES.items():
-                if candidate.endswith(optional):
+                if re.match(optional, candidate):
                     for distribution_name, specifier in dependencies.items():
                         result, reason = match_optional_dependencies(distribution_name, specifier)
                         if not result:

tests/providers/opensearch/conftest.py

@@ -19,12 +19,19 @@
 from typing import Any
 
 import pytest
-from opensearchpy import OpenSearch
 
+from airflow.hooks.base import BaseHook
 from airflow.models import Connection
-from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook
 from airflow.utils import db
 
+try:
+    from opensearchpy import OpenSearch
+
+    from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook
+except ImportError:
+    OpenSearch = None  # type: ignore[assignment, misc]
+    OpenSearchHook = BaseHook  # type: ignore[assignment,misc]
+
 # TODO: FIXME - those Mocks have overrides that are not used but they also do not make Mypy Happy
 # mypy: disable-error-code="override"
 

tests/providers/opensearch/hooks/test_opensearch.py

@@ -18,8 +18,9 @@
 
 from unittest import mock
 
-import opensearchpy
 import pytest
+
+opensearchpy = pytest.importorskip("opensearchpy")
 from opensearchpy import Urllib3HttpConnection
 
 from airflow.exceptions import AirflowException

tests/providers/opensearch/operators/test_opensearch.py

@@ -17,6 +17,9 @@
 from __future__ import annotations
 
 import pytest
+
+opensearchpy = pytest.importorskip("opensearchpy")
+
 from opensearchpy import Document, Keyword, Text
 
 from airflow.models import DAG