🚨 Snowflake produces permanent tables 🚨 (airbytehq#9063)

LOVEGISER · Jan 6, 2022 · b6926d4 · b6926d4
1 parent c5d4a97
commit b6926d4
Show file tree

Hide file tree

Showing 33 changed files with 224 additions and 60 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -179,7 +179,7 @@
 - name: Snowflake
   destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   dockerRepository: airbyte/destination-snowflake
-  dockerImageTag: 0.3.24
+  dockerImageTag: 0.4.0
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
   icon: snowflake.svg
 - name: MariaDB ColumnStore

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -3767,7 +3767,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-snowflake:0.3.24"
+- dockerImage: "airbyte/destination-snowflake:0.4.0"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake"
     connectionSpecification:

diff --git a/airbyte-integrations/bases/base-normalization/.dockerignore b/airbyte-integrations/bases/base-normalization/.dockerignore
@@ -9,3 +9,4 @@
 !dbt-project-template-mysql
 !dbt-project-template-oracle
 !dbt-project-template-clickhouse
+!dbt-project-template-snowflake
diff --git a/airbyte-integrations/bases/base-normalization/.gitignore b/airbyte-integrations/bases/base-normalization/.gitignore
@@ -19,20 +19,31 @@ integration_tests/normalization_test_output/**/*.yml
 # We keep a minimal/restricted subset of sql files for all destinations to avoid noise in diff
 # Simple Streams
 !integration_tests/normalization_test_output/**/dedup_exchange_rate*.sql
+!integration_tests/normalization_test_output/**/DEDUP_EXCHANGE_RATE*.sql
 !integration_tests/normalization_test_output/**/exchange_rate.sql
+!integration_tests/normalization_test_output/**/EXCHANGE_RATE.sql
 !integration_tests/normalization_test_output/**/test_simple_streams/first_output/airbyte_views/**/multiple_column_names_conflicts_stg.sql
 # Nested Streams
 # Parent table
 !integration_tests/normalization_test_output/**/nested_stream_with*_names_ab*.sql
 !integration_tests/normalization_test_output/**/nested_stream_with*_names_scd.sql
 !integration_tests/normalization_test_output/**/nested_stream_with*_names.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES_AB*.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES_SCD.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES.sql
 # Nested table
 !integration_tests/normalization_test_output/**/nested_stream_with_*_partition_ab1.sql
 !integration_tests/normalization_test_output/**/nested_stream_with_*_data_ab1.sql
 !integration_tests/normalization_test_output/**/nested_stream_with*_partition_scd.sql
 !integration_tests/normalization_test_output/**/nested_stream_with*_data_scd.sql
 !integration_tests/normalization_test_output/**/nested_stream_with*_partition.sql
 !integration_tests/normalization_test_output/**/nested_stream_with*_data.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH_*_PARTITION_AB1.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH_*_DATA_AB1.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_PARTITION_SCD.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_DATA_SCD.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_PARTITION.sql
+!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_DATA.sql
 
 # but we keep all sql files for Postgres
 !integration_tests/normalization_test_output/postgres/**/*.sql

diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle
@@ -73,11 +73,16 @@ task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) {
     configure buildAirbyteDocker('clickhouse')
     dependsOn assemble
 }
+task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) {
+    configure buildAirbyteDocker('snowflake')
+    dependsOn assemble
+}
 
 airbyteDocker.dependsOn(airbyteDockerMSSql)
 airbyteDocker.dependsOn(airbyteDockerMySql)
 airbyteDocker.dependsOn(airbyteDockerOracle)
 airbyteDocker.dependsOn(airbyteDockerClickhouse)
+airbyteDocker.dependsOn(airbyteDockerSnowflake)
 
 task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) {
     module = "pytest"

diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-snowflake/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-snowflake/dbt_project.yml
@@ -0,0 +1,64 @@
+# This file is necessary to install dbt-utils with dbt deps
+# the content will be overwritten by the transform function
+
+# Name your package! Package names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: "airbyte_utils"
+version: "1.0"
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project. Profiles contain
+# database connection information, and should be configured in the  ~/.dbt/profiles.yml file
+profile: "normalize"
+
+# These configurations specify where dbt should look for different types of files.
+# The `source-paths` config, for example, states that source models can be found
+# in the "models/" directory. You probably won't need to change these!
+source-paths: ["models"]
+docs-paths: ["docs"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+data-paths: ["data"]
+macro-paths: ["macros"]
+
+target-path: "../build" # directory which will store compiled SQL files
+log-path: "../logs" # directory which will store DBT logs
+modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+
+clean-targets: # directories to be removed by `dbt clean`
+  - "build"
+  - "dbt_modules"
+
+quoting:
+  database: true
+  # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
+  # all schemas should be unquoted
+  schema: false
+  identifier: true
+
+# You can define configurations for models in the `source-paths` directory here.
+# Using these configurations, you can enable or disable models, change how they
+# are materialized, and more!
+models:
+  +transient: false
+  airbyte_utils:
+    +materialized: table
+    generated:
+      airbyte_ctes:
+        +tags: airbyte_internal_cte
+        +materialized: ephemeral
+      airbyte_incremental:
+        +tags: incremental_tables
+        +materialized: incremental
+        +on_schema_change: sync_all_columns
+      airbyte_tables:
+        +tags: normalized_tables
+        +materialized: table
+      airbyte_views:
+        +tags: airbyte_internal_views
+        +materialized: view
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ["airbyte_utils", "dbt_utils"]
diff --git a/airbyte-integrations/bases/base-normalization/docker-compose.build.yaml b/airbyte-integrations/bases/base-normalization/docker-compose.build.yaml
@@ -36,3 +36,10 @@ services:
       context: .
       labels:
         io.airbyte.git-revision: ${GIT_REVISION}
+  normalization-snowflake:
+    image: airbyte/normalization-snowflake:${VERSION}
+    build:
+      dockerfile: snowflake.Dockerfile
+      context: .
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
diff --git a/airbyte-integrations/bases/base-normalization/docker-compose.yaml b/airbyte-integrations/bases/base-normalization/docker-compose.yaml
@@ -12,3 +12,5 @@ services:
     image: airbyte/normalization-oracle:${VERSION}
   normalization-clickhouse:
     image: airbyte/normalization-clickhouse:${VERSION}
+  normalization-snowflake:
+    image: airbyte/normalization-snowflake:${VERSION}
diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py
@@ -374,6 +374,8 @@ def get_normalization_image(destination_type: DestinationType) -> str:
             return "airbyte/normalization-oracle:dev"
         elif DestinationType.CLICKHOUSE.value == destination_type.value:
             return "airbyte/normalization-clickhouse:dev"
+        elif DestinationType.SNOWFLAKE.value == destination_type.value:
+            return "airbyte/normalization-snowflake:dev"
         else:
             return "airbyte/normalization:dev"
 
@@ -445,6 +447,8 @@ def run_check_dbt_command(normalization_image: str, command: str, cwd: str, forc
                         "Configuration paths exist in your dbt_project.yml",  # When no cte / view are generated
                         "Error loading config file: .dockercfg: $HOME is not defined",  # ignore warning
                         "depends on a node named 'disabled_test' which was not found",  # Tests throwing warning because it is disabled
+                        "The requested image's platform (linux/amd64) does not match the detected host platform "
+                        + "(linux/arm64/v8) and no specific platform was requested",  # temporary patch until we publish images for arm64
                     ]:
                         if except_clause in str_line:
                             is_exception = True

diff --git a/...integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml b/...integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml
@@ -4,13 +4,13 @@
 # Name your package! Package names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'airbyte_utils'
-version: '1.0'
+name: "airbyte_utils"
+version: "1.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project. Profiles contain
 # database connection information, and should be configured in the  ~/.dbt/profiles.yml file
-profile: 'normalize'
+profile: "normalize"
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that source models can be found
@@ -22,25 +22,26 @@ test-paths: ["tests"]
 data-paths: ["data"]
 macro-paths: ["macros"]
 
-target-path: "../build"  # directory which will store compiled SQL files
-log-path: "../logs"  # directory which will store DBT logs
-modules-path: "/tmp/dbt_modules"  # directory which will store external DBT dependencies
+target-path: "../build" # directory which will store compiled SQL files
+log-path: "../logs" # directory which will store DBT logs
+modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
 
-clean-targets:         # directories to be removed by `dbt clean`
-    - "build"
-    - "dbt_modules"
+clean-targets: # directories to be removed by `dbt clean`
+  - "build"
+  - "dbt_modules"
 
 quoting:
   database: true
-# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
-# all schemas should be unquoted
+  # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
+  # all schemas should be unquoted
   schema: false
   identifier: true
 
 # You can define configurations for models in the `source-paths` directory here.
 # Using these configurations, you can enable or disable models, change how they
 # are materialized, and more!
 models:
+  +transient: false
   airbyte_utils:
     +materialized: table
     generated:
@@ -60,4 +61,4 @@ models:
 
 dispatch:
   - macro_namespace: dbt_utils
-    search_order: ['airbyte_utils', 'dbt_utils']
+    search_order: ["airbyte_utils", "dbt_utils"]
diff --git a/...ental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/...ental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES"  as
       (select * from(
 
 -- Final base SQL model

diff --git a/..._NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/..._NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION"  as
       (select * from(
 
 with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1 as (

diff --git a/...ALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/...ALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA"  as
       (select * from(
 
 with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1 as (

diff --git a/...TED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/...TED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA"  as
       (select * from(
 
 with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1 as (

diff --git a/...d/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/...d/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD"  as
       (select * from(
 
 -- depends_on: ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_STG')

diff --git a/...integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml b/...integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml
@@ -4,13 +4,13 @@
 # Name your package! Package names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'airbyte_utils'
-version: '1.0'
+name: "airbyte_utils"
+version: "1.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project. Profiles contain
 # database connection information, and should be configured in the  ~/.dbt/profiles.yml file
-profile: 'normalize'
+profile: "normalize"
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that source models can be found
@@ -22,25 +22,26 @@ test-paths: ["tests"]
 data-paths: ["data"]
 macro-paths: ["macros"]
 
-target-path: "../build"  # directory which will store compiled SQL files
-log-path: "../logs"  # directory which will store DBT logs
-modules-path: "/tmp/dbt_modules"  # directory which will store external DBT dependencies
+target-path: "../build" # directory which will store compiled SQL files
+log-path: "../logs" # directory which will store DBT logs
+modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
 
-clean-targets:         # directories to be removed by `dbt clean`
-    - "build"
-    - "dbt_modules"
+clean-targets: # directories to be removed by `dbt clean`
+  - "build"
+  - "dbt_modules"
 
 quoting:
   database: true
-# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
-# all schemas should be unquoted
+  # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
+  # all schemas should be unquoted
   schema: false
   identifier: true
 
 # You can define configurations for models in the `source-paths` directory here.
 # Using these configurations, you can enable or disable models, change how they
 # are materialized, and more!
 models:
+  +transient: false
   airbyte_utils:
     +materialized: table
     generated:
@@ -60,4 +61,4 @@ models:
 
 dispatch:
   - macro_namespace: dbt_utils
-    search_order: ['airbyte_utils', 'dbt_utils']
+    search_order: ["airbyte_utils", "dbt_utils"]
diff --git a/...imple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/...imple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE"  as
       (select * from(
 
 -- Final base SQL model

diff --git a/...reams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/...reams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD"  as
       (select * from(
 
 -- depends_on: ref('DEDUP_EXCHANGE_RATE_STG')

diff --git a/...lake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/...lake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE"  as
       (select * from(
 
 with __dbt__cte__EXCHANGE_RATE_AB1 as (

diff --git a/...ake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/...ake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql
@@ -1,6 +1,6 @@
 
 
-      create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE"  as
+      create or replace  table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE"  as
       (select * from(
 
 with __dbt__cte__EXCHANGE_RATE_AB1 as (

diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py
@@ -134,6 +134,8 @@ def setup_test_dir(integration_type: str) -> str:
         copy_tree("../dbt-project-template-mysql", test_root_dir)
     elif integration_type == DestinationType.ORACLE.value:
         copy_tree("../dbt-project-template-oracle", test_root_dir)
+    elif integration_type == DestinationType.SNOWFLAKE.value:
+        copy_tree("../dbt-project-template-snowflake", test_root_dir)
     return test_root_dir
 
 

diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py
@@ -192,6 +192,9 @@ def setup_test_dir(destination_type: DestinationType, test_resource_name: str) -
     elif destination_type.value == DestinationType.CLICKHOUSE.value:
         copy_tree("../dbt-project-template-clickhouse", test_root_dir)
         dbt_project_yaml = "../dbt-project-template-clickhouse/dbt_project.yml"
+    elif destination_type.value == DestinationType.SNOWFLAKE.value:
+        copy_tree("../dbt-project-template-snowflake", test_root_dir)
+        dbt_project_yaml = "../dbt-project-template-snowflake/dbt_project.yml"
     dbt_test_utils.copy_replace(dbt_project_yaml, os.path.join(test_root_dir, "dbt_project.yml"))
     return test_root_dir