diff --git a/pytest.ini b/pytest.ini index b04a6ccf3..28cefdb3e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,3 +8,7 @@ testpaths = tests/unit tests/integration tests/functional +markers = + external: mark test as requiring an external location + python: mark test as running a python model + dlt: mark test as running a DLT model \ No newline at end of file diff --git a/tests/functional/adapter/basic/test_ensure_no_describe_extended.py b/tests/functional/adapter/basic/test_ensure_no_describe_extended.py deleted file mode 100644 index 58d989c49..000000000 --- a/tests/functional/adapter/basic/test_ensure_no_describe_extended.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest - -from dbt.tests import util -from tests.functional.adapter.basic import fixtures - - -class TestEnsureNoDescribeExtended: - """Tests in this class exist to ensure we don't call describe extended unnecessarily. - This became a problem due to needing to discern tables from streaming tables, which is not - relevant on hive, but users on hive were having all of their tables describe extended-ed. - We only need to call describe extended if we are using a UC catalog and we can't determine the - type of the materialization.""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"my_seed.csv": fixtures.basic_seed_csv} - - @pytest.fixture(scope="class") - def models(self): - return {"my_model.sql": fixtures.basic_model_sql} - - def test_ensure_no_describe_extended(self, project): - # Add some existing data to ensure we don't try to 'describe extended' it. - util.run_dbt(["seed"]) - - _, log_output = util.run_dbt_and_capture(["run"]) - assert "describe extended" not in log_output diff --git a/tests/functional/adapter/basic/test_incremental.py b/tests/functional/adapter/basic/test_incremental.py index 73c99260e..8d630a0a9 100644 --- a/tests/functional/adapter/basic/test_incremental.py +++ b/tests/functional/adapter/basic/test_incremental.py @@ -24,6 +24,7 @@ class TestIncrementalDeltaNotSchemaChange(BaseIncrementalNotSchemaChange): pass +@pytest.mark.external @pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster") class TestIncrementalParquet(BaseIncremental): @pytest.fixture(scope="class") @@ -50,6 +51,7 @@ def project_config_update(self): } +@pytest.mark.external @pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster") class TestIncrementalCSV(BaseIncremental): @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/incremental/test_incremental_clustering.py b/tests/functional/adapter/incremental/test_incremental_clustering.py index 01885406a..cb867ade3 100644 --- a/tests/functional/adapter/incremental/test_incremental_clustering.py +++ b/tests/functional/adapter/incremental/test_incremental_clustering.py @@ -36,6 +36,7 @@ def test_changing_cluster_by(self, project): assert False +@pytest.mark.python @pytest.mark.skip_profile("databricks_cluster") class TestIncrementalPythonLiquidClustering: @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/incremental/test_incremental_strategies.py b/tests/functional/adapter/incremental/test_incremental_strategies.py index e941adcf2..99ac3ff40 100644 --- a/tests/functional/adapter/incremental/test_incremental_strategies.py +++ b/tests/functional/adapter/incremental/test_incremental_strategies.py @@ -42,6 +42,7 @@ class TestAppendDelta(AppendBase): pass +@pytest.mark.external @pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster") class TestAppendParquet(AppendBase): @pytest.fixture(scope="class") @@ -118,6 +119,7 @@ def test_incremental(self, project): util.check_relations_equal(project.adapter, ["overwrite_model", "upsert_expected"]) +@pytest.mark.external @pytest.mark.skip("This test is not repeatable due to external location") class TestInsertOverwriteParquet(InsertOverwriteBase): @pytest.fixture(scope="class") @@ -132,6 +134,7 @@ def project_config_update(self): } +@pytest.mark.external @pytest.mark.skip("This test is not repeatable due to external location") class TestInsertOverwriteWithPartitionsParquet(InsertOverwriteBase): @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/incremental/test_incremental_tags.py b/tests/functional/adapter/incremental/test_incremental_tags.py index 730d9549f..b84e55e1f 100644 --- a/tests/functional/adapter/incremental/test_incremental_tags.py +++ b/tests/functional/adapter/incremental/test_incremental_tags.py @@ -28,6 +28,7 @@ def test_changing_tags(self, project): assert results_dict == {"c": "e", "d": "f"} +@pytest.mark.python @pytest.mark.skip_profile("databricks_cluster") class TestIncrementalPythonTags: @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/incremental/test_incremental_tblproperties.py b/tests/functional/adapter/incremental/test_incremental_tblproperties.py index 22625a9b6..2893067f7 100644 --- a/tests/functional/adapter/incremental/test_incremental_tblproperties.py +++ b/tests/functional/adapter/incremental/test_incremental_tblproperties.py @@ -27,6 +27,7 @@ def test_changing_tblproperties(self, project): assert results_dict["d"] == "f" +@pytest.mark.python @pytest.mark.skip_profile("databricks_cluster") class TestIncrementalPythonTblproperties: @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/long_sessions/fixtures.py b/tests/functional/adapter/long_sessions/fixtures.py deleted file mode 100644 index f1332e877..000000000 --- a/tests/functional/adapter/long_sessions/fixtures.py +++ /dev/null @@ -1,46 +0,0 @@ -source = """id,name,date -1,Alice,2022-01-01 -2,Bob,2022-01-02 -""" - -target = """ -{{config(materialized='table')}} - -select * from {{ ref('source') }} -""" - -target2 = """ -{{config(materialized='table', databricks_compute='alternate_warehouse')}} - -select * from {{ ref('source') }} -""" - -targetseq1 = """ -{{config(materialized='table', databricks_compute='alternate_warehouse')}} - -select * from {{ ref('source') }} -""" - -targetseq2 = """ -{{config(materialized='table')}} - -select * from {{ ref('targetseq1') }} -""" - -targetseq3 = """ -{{config(materialized='table')}} - -select * from {{ ref('targetseq2') }} -""" - -targetseq4 = """ -{{config(materialized='table')}} - -select * from {{ ref('targetseq3') }} -""" - -targetseq5 = """ -{{config(materialized='table', databricks_compute='alternate_warehouse')}} - -select * from {{ ref('targetseq4') }} -""" diff --git a/tests/functional/adapter/long_sessions/test_long_sessions.py b/tests/functional/adapter/long_sessions/test_long_sessions.py deleted file mode 100644 index 2f7e8065d..000000000 --- a/tests/functional/adapter/long_sessions/test_long_sessions.py +++ /dev/null @@ -1,101 +0,0 @@ -import os -from unittest import mock - -import pytest -from dbt.tests import util -from tests.functional.adapter.long_sessions import fixtures - -with mock.patch.dict( - os.environ, - { - "DBT_DATABRICKS_LONG_SESSIONS": "true", - "DBT_DATABRICKS_CONNECTOR_LOG_LEVEL": "DEBUG", - }, -): - import dbt.adapters.databricks.connections # noqa - - -class TestLongSessionsBase: - args_formatter = "" - - @pytest.fixture(scope="class") - def seeds(self): - return { - "source.csv": fixtures.source, - } - - @pytest.fixture(scope="class") - def models(self): - m = {} - for i in range(5): - m[f"target{i}.sql"] = fixtures.target - - return m - - def test_long_sessions(self, project): - _, log = util.run_dbt_and_capture(["--debug", "seed"]) - open_count = log.count("request: OpenSession") - assert open_count == 2 - - _, log = util.run_dbt_and_capture(["--debug", "run"]) - open_count = log.count("request: OpenSession") - assert open_count == 2 - - -class TestLongSessionsMultipleThreads(TestLongSessionsBase): - def test_long_sessions(self, project): - util.run_dbt_and_capture(["seed"]) - - for n_threads in [1, 2, 3]: - _, log = util.run_dbt_and_capture(["--debug", "run", "--threads", f"{n_threads}"]) - open_count = log.count("request: OpenSession") - assert open_count == (n_threads + 1) - - -class TestLongSessionsMultipleCompute: - args_formatter = "" - - @pytest.fixture(scope="class") - def seeds(self): - return { - "source.csv": fixtures.source, - } - - @pytest.fixture(scope="class") - def models(self): - m = {} - for i in range(2): - m[f"target{i}.sql"] = fixtures.target - - m["target_alt.sql"] = fixtures.target2 - - return m - - def test_long_sessions(self, project): - util.run_dbt_and_capture(["--debug", "seed", "--target", "alternate_warehouse"]) - - _, log = util.run_dbt_and_capture(["--debug", "run", "--target", "alternate_warehouse"]) - open_count = log.count("request: OpenSession") - assert open_count == 3 - - -class TestLongSessionsIdleCleanup(TestLongSessionsMultipleCompute): - args_formatter = "" - - @pytest.fixture(scope="class") - def models(self): - m = { - "targetseq1.sql": fixtures.targetseq1, - "targetseq2.sql": fixtures.targetseq2, - "targetseq3.sql": fixtures.targetseq3, - "targetseq4.sql": fixtures.targetseq4, - "targetseq5.sql": fixtures.targetseq5, - } - return m - - def test_long_sessions(self, project): - util.run_dbt(["--debug", "seed", "--target", "idle_sessions"]) - - _, log = util.run_dbt_and_capture(["--debug", "run", "--target", "idle_sessions"]) - idle_count = log.count("Closing for idleness") / 2 - assert idle_count > 0 diff --git a/tests/functional/adapter/materialized_view_tests/test_basic.py b/tests/functional/adapter/materialized_view_tests/test_basic.py index 22b74ac09..08c49bba8 100644 --- a/tests/functional/adapter/materialized_view_tests/test_basic.py +++ b/tests/functional/adapter/materialized_view_tests/test_basic.py @@ -27,6 +27,7 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]: return fixtures.query_relation_type(project, relation) +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestMaterializedViews(TestMaterializedViewsMixin, MaterializedViewBasic): def test_table_replaces_materialized_view(self, project, my_materialized_view): diff --git a/tests/functional/adapter/materialized_view_tests/test_changes.py b/tests/functional/adapter/materialized_view_tests/test_changes.py index a113d2fb7..9a123638a 100644 --- a/tests/functional/adapter/materialized_view_tests/test_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_changes.py @@ -80,6 +80,7 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]: return fixtures.query_relation_type(project, relation) +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestMaterializedViewApplyChanges( MaterializedViewChangesMixin, MaterializedViewChangesApplyMixin @@ -87,6 +88,7 @@ class TestMaterializedViewApplyChanges( pass +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestMaterializedViewContinueOnChanges( MaterializedViewChangesMixin, MaterializedViewChangesContinueMixin @@ -94,6 +96,7 @@ class TestMaterializedViewContinueOnChanges( pass +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestMaterializedViewFailOnChanges( MaterializedViewChangesMixin, MaterializedViewChangesFailMixin diff --git a/tests/functional/adapter/persist_docs/test_persist_docs.py b/tests/functional/adapter/persist_docs/test_persist_docs.py index cbc7d0a12..2544b1015 100644 --- a/tests/functional/adapter/persist_docs/test_persist_docs.py +++ b/tests/functional/adapter/persist_docs/test_persist_docs.py @@ -245,6 +245,7 @@ def test_quoted_column_comments(self, adapter, table_relation): break +@pytest.mark.external # Skipping UC Cluster to ensure these tests don't fail due to overlapping resources @pytest.mark.skip_profile("databricks_uc_cluster") class TestPersistDocsWithSeeds: diff --git a/tests/functional/adapter/python_model/test_python_model.py b/tests/functional/adapter/python_model/test_python_model.py index a6e93a592..ad7541b24 100644 --- a/tests/functional/adapter/python_model/test_python_model.py +++ b/tests/functional/adapter/python_model/test_python_model.py @@ -9,16 +9,19 @@ from tests.functional.adapter.python_model import fixtures as override_fixtures +@pytest.mark.python @pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestPythonModel(BasePythonModelTests): pass +@pytest.mark.python @pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestPythonIncrementalModel(BasePythonIncrementalTests): pass +@pytest.mark.python @pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestChangingSchema: @pytest.fixture(scope="class") @@ -42,6 +45,7 @@ def test_changing_schema_with_log_validation(self, project, logs_dir): assert "Execution status: OK in" in log +@pytest.mark.python @pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestChangingSchemaIncremental: @pytest.fixture(scope="class") @@ -60,6 +64,7 @@ def test_changing_schema_via_incremental(self, project): util.check_relations_equal(project.adapter, ["incremental_model", "expected_incremental"]) +@pytest.mark.python @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestSpecifyingHttpPath(BasePythonModelTests): @pytest.fixture(scope="class") @@ -73,6 +78,8 @@ def models(self): } +@pytest.mark.python +@pytest.mark.external @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_sql_endpoint") class TestComplexConfig: @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/python_model/test_spark.py b/tests/functional/adapter/python_model/test_spark.py index 222fb60a3..876098973 100644 --- a/tests/functional/adapter/python_model/test_spark.py +++ b/tests/functional/adapter/python_model/test_spark.py @@ -7,6 +7,7 @@ ) +@pytest.mark.python @pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestPySpark(BasePySparkTests): @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/simple_copy/test_simple_copy.py b/tests/functional/adapter/simple_copy/test_simple_copy.py index a7328d991..feac39bd5 100644 --- a/tests/functional/adapter/simple_copy/test_simple_copy.py +++ b/tests/functional/adapter/simple_copy/test_simple_copy.py @@ -5,6 +5,7 @@ # Tests with materialized_views, which only works for SQL Warehouse +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestSimpleCopyBase(SimpleCopyBase): pass diff --git a/tests/functional/adapter/streaming_tables/test_st_basic.py b/tests/functional/adapter/streaming_tables/test_st_basic.py index cf26ba854..ad5d8a253 100644 --- a/tests/functional/adapter/streaming_tables/test_st_basic.py +++ b/tests/functional/adapter/streaming_tables/test_st_basic.py @@ -12,6 +12,7 @@ from tests.functional.adapter.streaming_tables import fixtures +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestStreamingTablesBasic: @staticmethod diff --git a/tests/functional/adapter/streaming_tables/test_st_changes.py b/tests/functional/adapter/streaming_tables/test_st_changes.py index 3791bd698..c0495f359 100644 --- a/tests/functional/adapter/streaming_tables/test_st_changes.py +++ b/tests/functional/adapter/streaming_tables/test_st_changes.py @@ -123,6 +123,7 @@ def test_full_refresh_occurs_with_changes(self, project, my_streaming_table): util.assert_message_in_logs(f"Applying REPLACE to: {my_streaming_table}", logs) +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestStreamingTableChangesApply(StreamingTableChanges): @pytest.fixture(scope="class") @@ -153,6 +154,7 @@ def test_change_is_applied_via_replace(self, project, my_streaming_table): util.assert_message_in_logs(f"Applying REPLACE to: {my_streaming_table}", logs) +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestStreamingTableChangesContinue(StreamingTableChanges): @pytest.fixture(scope="class") @@ -193,6 +195,7 @@ def test_change_is_not_applied_via_replace(self, project, my_streaming_table): util.assert_message_in_logs(f"Applying REPLACE to: {my_streaming_table}", logs, False) +@pytest.mark.dlt @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster") class TestStreamingTableChangesFail(StreamingTableChanges): @pytest.fixture(scope="class") diff --git a/tests/functional/adapter/tags/test_databricks_tags.py b/tests/functional/adapter/tags/test_databricks_tags.py index b52c7caed..2221a11b8 100644 --- a/tests/functional/adapter/tags/test_databricks_tags.py +++ b/tests/functional/adapter/tags/test_databricks_tags.py @@ -36,6 +36,7 @@ def models(self): return {"tags.sql": fixtures.tags_sql.replace("table", "incremental")} +@pytest.mark.python @pytest.mark.skip_profile("databricks_cluster") class TestPythonTags(TestTags): @pytest.fixture(scope="class") diff --git a/tests/profiles.py b/tests/profiles.py index 525d51c6a..c024e12c7 100644 --- a/tests/profiles.py +++ b/tests/profiles.py @@ -44,10 +44,6 @@ def _build_databricks_cluster_target( "_port": os.getenv("DBT_DATABRICKS_PORT"), "_tls_no_verify": True, } - if os.getenv("DBT_DATABRICKS_CA_PATH"): - profile["connection_parameters"]["_tls_trusted_ca_file"] = os.getenv( - "DBT_DATABRICKS_CA_PATH" - ) return profile