Merge branch 'main' into 1.9.latest

.github/workflows/integration.yml

@@ -8,6 +8,11 @@ on:
       - "tests/unit/**"
       - ".github/workflows/main.yml"
       - ".github/workflows/stale.yml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   run-tox-tests-uc-cluster:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -21,9 +21,14 @@
 
 ## dbt-databricks 1.8.7 (TBD)
 
+### Features
+
+- Add config for generating unique tmp table names for enabling parralel replace-where (thanks @huangxingyi-git!) ([811](https://github.com/databricks/dbt-databricks/pull/811))
+
 ### Fixes
 
 - Stop setting cluster by to None. If you want to drop liquid clustering, you will need to full-refresh ([806]https://github.com/databricks/dbt-databricks/pull/806)
+- Don't define table properties on snapshot staging views (thanks @jelmerk!) ([820](https://github.com/databricks/dbt-databricks/pull/820))
 
 ## dbt-databricks 1.8.6 (September 18, 2024)
 

dbt/adapters/databricks/connections.py

@@ -77,6 +77,7 @@
 from dbt_common.exceptions import DbtDatabaseError
 from dbt_common.exceptions import DbtInternalError
 from dbt_common.exceptions import DbtRuntimeError
+from dbt_common.exceptions import DbtDatabaseError
 from dbt_common.utils import cast_to_str
 from requests import Session
 

dbt/adapters/databricks/impl.py

@@ -22,6 +22,7 @@
 from typing import Type
 from typing import TYPE_CHECKING
 from typing import Union
+from uuid import uuid4
 
 from dbt.adapters.base import AdapterConfig
 from dbt.adapters.base import PythonJobHelper
@@ -121,6 +122,7 @@ class DatabricksConfig(AdapterConfig):
     databricks_tags: Optional[Dict[str, str]] = None
     tblproperties: Optional[Dict[str, str]] = None
     zorder: Optional[Union[List[str], str]] = None
+    unique_tmp_table_suffix: bool = False
     skip_non_matched_step: Optional[bool] = None
     skip_matched_step: Optional[bool] = None
     matched_condition: Optional[str] = None
@@ -707,6 +709,10 @@ def get_config_from_model(self, model: RelationConfig) -> DatabricksRelationConf
                 f"Materialization {model.config.materialized} is not supported."
             )
 
+    @available
+    def generate_unique_temporary_table_suffix(self, suffix_initial: str = "__dbt_tmp") -> str:
+        return f"{suffix_initial}_{str(uuid4())}"
+
 
 @dataclass(frozen=True)
 class RelationAPIBase(ABC, Generic[DatabricksRelationConfig]):

dbt/include/databricks/macros/materializations/incremental/incremental.sql

@@ -5,6 +5,7 @@
   {%- set grant_config = config.get('grants') -%}
   {%- set tblproperties = config.get('tblproperties') -%}
   {%- set tags = config.get('databricks_tags') -%}
+  {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%}
 
   {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%}
   {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%}
@@ -18,7 +19,11 @@
   {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
   {%- set target_relation = this.incorporate(type='table') -%}
   {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}
-
+  {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%}
+    {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%}
+  {%- else -%}
+    {%- set temp_relation_suffix = '__dbt_tmp' -%}
+  {%- endif -%}
 
   {#-- Set Overwrite Mode to STATIC for initial replace --#}
   {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%}
@@ -69,7 +74,7 @@
     {%- set _existing_config = adapter.get_relation_config(existing_relation) -%}
     {%- set model_config = adapter.get_config_from_model(config.model) -%}
     {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}
-    {%- set temp_relation = databricks__make_temp_relation(target_relation, as_table=language != 'sql') -%}
+    {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%}
     {%- call statement('create_temp_relation', language=language) -%}
       {{ create_table_as(True, temp_relation, compiled_code, language) }}
     {%- endcall -%}

dbt/include/databricks/macros/materializations/snapshot.sql

@@ -10,7 +10,9 @@
 
     {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}
     {% call statement('build_snapshot_staging_relation') %}
-        {{ create_view_as(tmp_relation, select) }}
+        create or replace view {{ tmp_relation }}
+        as
+            {{ select }}
     {% endcall %}
 
     {% do return(tmp_relation) %}