Skip to content

Drop provider dependency from airflow-core secrets backend tests #71677

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rjgoyln wants to merge 1 commit into
base: main
Choose a base branch
from
+113 −145
Draft

Drop provider dependency from airflow-core secrets backend tests #71677

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 27 additions & 71 deletions airflow-core/tests/unit/always/test_secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@

from tests_common.test_utils.config import conf_vars
from tests_common.test_utils.db import clear_db_variables
from tests_common.test_utils.markers import skip_if_force_lowest_dependencies_marker
from tests_common.test_utils.fake_secrets_backend import FAKE_BACKEND_PATH, FakeSecretsBackend


@skip_if_force_lowest_dependencies_marker
class TestConnectionsFromSecrets:
def setup_method(self) -> None:
SecretCache.reset()
Expand All @@ -55,46 +54,35 @@ def test_get_connection_first_try(self, mock_env_get, mock_meta_get):

@conf_vars(
{
(
"secrets",
"backend",
): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow", "profile_name": null}',
("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow"}',
}
)
def test_initialize_secrets_backends(self):
backends = initialize_secrets_backends()
backend_classes = [backend.__class__.__name__ for backend in backends]

assert len(backends) == 3
assert "SystemsManagerParameterStoreBackend" in backend_classes
assert "FakeSecretsBackend" in backend_classes

@conf_vars(
{
(
"secrets",
"backend",
): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"use_ssl": false}',
}
)
def test_backends_kwargs(self):
backends = initialize_secrets_backends()
systems_manager = next(
backend
for backend in backends
if backend.__class__.__name__ == "SystemsManagerParameterStoreBackend"
custom_backend = next(
backend for backend in backends if backend.__class__.__name__ == "FakeSecretsBackend"
)
assert systems_manager.kwargs == {}
assert systems_manager.use_ssl is False
assert custom_backend.kwargs == {}
assert custom_backend.use_ssl is False

@conf_vars(
{
(
"secrets",
"backend",
): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow", "profile_name": null}',
("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow"}',
}
)
@mock.patch.dict(
Expand All @@ -103,20 +91,16 @@ def test_backends_kwargs(self):
"AIRFLOW_CONN_TEST_MYSQL": "mysql://airflow:airflow@host:5432/airflow",
},
)
@mock.patch(
"airflow.providers.amazon.aws.secrets.systems_manager."
"SystemsManagerParameterStoreBackend.get_connection"
)
@mock.patch.object(FakeSecretsBackend, "get_connection")
def test_backend_fallback_to_env_var(self, mock_get_connection):
mock_get_connection.return_value = None

backends = ensure_secrets_loaded()
backend_classes = [backend.__class__.__name__ for backend in backends]
assert "SystemsManagerParameterStoreBackend" in backend_classes
assert "FakeSecretsBackend" in backend_classes

conn = Connection.get_connection_from_secrets(conn_id="test_mysql")

# Assert that SystemsManagerParameterStoreBackend.get_conn_uri was called
mock_get_connection.assert_called_once_with(conn_id="test_mysql", team_name=None)

assert conn.get_uri() == "mysql://airflow:airflow@host:5432/airflow"
Expand All @@ -134,7 +118,6 @@ def test_connection_env_var_do_not_access_team_specific(self):
Connection.get_connection_from_secrets(conn_id="_team___test_mysql")


@skip_if_force_lowest_dependencies_marker
@pytest.mark.db_test
class TestVariableFromSecrets:
def setup_method(self) -> None:
Expand Down Expand Up @@ -181,11 +164,8 @@ def test_backend_fallback_to_default_var(self):

@conf_vars(
{
(
"secrets",
"backend",
): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
("secrets", "backend_kwargs"): '{"variables_prefix": "/airflow", "profile_name": null}',
("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"variables_prefix": "/airflow"}',
}
)
@mock.patch.dict(
Expand All @@ -195,14 +175,11 @@ def test_backend_fallback_to_default_var(self):
},
)
@mock.patch("airflow.secrets.metastore.MetastoreBackend.get_variable")
@mock.patch(
"airflow.providers.amazon.aws.secrets.systems_manager."
"SystemsManagerParameterStoreBackend.get_variable"
)
@mock.patch.object(FakeSecretsBackend, "get_variable")
def test_backend_variable_order(self, mock_secret_get, mock_meta_get):
backends = ensure_secrets_loaded()
backend_classes = [backend.__class__.__name__ for backend in backends]
assert "SystemsManagerParameterStoreBackend" in backend_classes
assert "FakeSecretsBackend" in backend_classes

mock_secret_get.return_value = None
mock_meta_get.return_value = None
Expand Down Expand Up @@ -250,7 +227,6 @@ def get_variable(self, key):
return self._vars.get(key)


@skip_if_force_lowest_dependencies_marker
class TestLegacyBackendSignatureCompat:
"""Backends whose overrides predate the ``team_name`` keyword must keep working (issue #1333)."""

Expand Down Expand Up @@ -279,39 +255,26 @@ def test_get_variable_with_legacy_get_variable_override(self, team_name):
assert value == "secret_value"


@skip_if_force_lowest_dependencies_marker
class TestSecretBackendKwargEnvVars:
"""Test per-key env var overrides for secrets backend kwargs."""

def setup_method(self) -> None:
SecretCache.reset()

@conf_vars(
{
(
"secrets",
"backend",
): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
}
)
@conf_vars({("secrets", "backend"): FAKE_BACKEND_PATH})
@mock.patch.dict(
"os.environ",
{"AIRFLOW__SECRETS__BACKEND_KWARG__CONNECTIONS_PREFIX": "/airflow/connections"},
)
def test_backend_kwarg_env_vars_basic(self):
"""Per-key env var is picked up when no JSON blob is set."""
backends = initialize_secrets_backends()
systems_manager = next(
b for b in backends if b.__class__.__name__ == "SystemsManagerParameterStoreBackend"
)
assert systems_manager.connections_prefix == "/airflow/connections"
custom_backend = next(b for b in backends if b.__class__.__name__ == "FakeSecretsBackend")
assert custom_backend.connections_prefix == "/airflow/connections"

@conf_vars(
{
(
"secrets",
"backend",
): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"connections_prefix": "/old"}',
}
)
Expand All @@ -322,17 +285,12 @@ def test_backend_kwarg_env_vars_basic(self):
def test_backend_kwarg_env_vars_override_json(self):
"""Per-key env var overrides the same key in the JSON blob."""
backends = initialize_secrets_backends()
systems_manager = next(
b for b in backends if b.__class__.__name__ == "SystemsManagerParameterStoreBackend"
)
assert systems_manager.connections_prefix == "/new"
custom_backend = next(b for b in backends if b.__class__.__name__ == "FakeSecretsBackend")
assert custom_backend.connections_prefix == "/new"

@conf_vars(
{
(
"secrets",
"backend",
): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow"}',
}
)
Expand All @@ -343,8 +301,6 @@ def test_backend_kwarg_env_vars_override_json(self):
def test_backend_kwarg_env_vars_merge_with_json(self):
"""Per-key env var is merged with (not replacing) the JSON blob."""
backends = initialize_secrets_backends()
systems_manager = next(
b for b in backends if b.__class__.__name__ == "SystemsManagerParameterStoreBackend"
)
assert systems_manager.connections_prefix == "/airflow"
assert systems_manager.variables_prefix == "/airflow/variables"
custom_backend = next(b for b in backends if b.__class__.__name__ == "FakeSecretsBackend")
assert custom_backend.connections_prefix == "/airflow"
assert custom_backend.variables_prefix == "/airflow/variables"
Loading
Loading