Destination DuckDB: Bump duckdb engine to 0.10.3 (#38576)

airbytehq · Jun 3, 2024 · f607d70 · f607d70
1 parent f4c69be
commit f607d70
Show file tree

Hide file tree

Showing 11 changed files with 288 additions and 147 deletions.
diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py
@@ -1,6 +1,4 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 import datetime
 import json
@@ -13,10 +11,12 @@
 
 import duckdb
 import pyarrow as pa
+
 from airbyte_cdk import AirbyteLogger
 from airbyte_cdk.destinations import Destination
 from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type
 
+
 logger = getLogger("airbyte")
 
 CONFIG_MOTHERDUCK_API_KEY = "motherduck_api_key"

diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/run.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/run.py
@@ -1,6 +1,4 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 import sys
 

diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py
@@ -1,6 +1,4 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 from __future__ import annotations
 
@@ -112,7 +110,7 @@ def table_schema() -> str:
 
 @pytest.fixture
 def configured_catalogue(
-    test_table_name: str, test_large_table_name: str, table_schema: str, 
+    test_table_name: str, test_large_table_name: str, table_schema: str,
 ) -> ConfiguredAirbyteCatalog:
     append_stream = ConfiguredAirbyteStream(
         stream=AirbyteStream(
@@ -261,17 +259,17 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab
                 record=AirbyteRecordMessage(
                     stream=table_name,
                     # Throw in empty nested objects and see how pyarrow deals with them.
-                    data={"key1": fake.first_name() , 
-                          "key2": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), 
-                          "nested1": {} if random.random()< 0.1 else { 
-                              "key3": fake.first_name() , 
-                              "key4": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), 
-                              "dictionary1":{} if random.random()< 0.1 else { 
-                                  "key3": fake.first_name() , 
+                    data={"key1": fake.first_name(),
+                          "key2": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999),
+                          "nested1": {} if random.random()< 0.1 else {
+                              "key3": fake.first_name(),
+                              "key4": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999),
+                              "dictionary1":{} if random.random()< 0.1 else {
+                                  "key3": fake.first_name(),
                                   "key4": "True" if random.random() < 0.5 else True
                                   }
                               }
-                          } 
+                          }
                           if random.random() < 0.9 else {},
 
                     emitted_at=int(datetime.now().timestamp()) * 1000,
@@ -284,8 +282,8 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab
 BATCH_WRITE_SIZE = 1000
 
 @pytest.mark.slow
-@pytest.mark.parametrize("airbyte_message_generator,explanation", 
-                         [(_airbyte_messages, "Test writing a large number of simple json objects."), 
+@pytest.mark.parametrize("airbyte_message_generator,explanation",
+                         [(_airbyte_messages, "Test writing a large number of simple json objects."),
                           (_airbyte_messages_with_inconsistent_json_fields, "Test writing a large number of json messages with inconsistent schema.")] )
 def test_large_number_of_writes(
     config: Dict[str, str],

diff --git a/airbyte-integrations/connectors/destination-duckdb/metadata.yaml b/airbyte-integrations/connectors/destination-duckdb/metadata.yaml
@@ -4,7 +4,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 94bd199c-2ff0-4aa2-b98e-17f0acb72610
-  dockerImageTag: 0.3.6
+  dockerImageTag: 0.4.0
   dockerRepository: airbyte/destination-duckdb
   githubIssueLabel: destination-duckdb
   icon: duckdb.svg
@@ -15,16 +15,21 @@ data:
       enabled: true
     oss:
       enabled: true
-  releaseStage: alpha
+  releaseStage: beta
   releases:
     breakingChanges:
       0.3.0:
-        message:
-          "This version uses the DuckDB 0.9.1 database driver, which is not
-          backwards compatible with prior versions. MotherDuck users can upgrade their
-          database by visiting https://app.motherduck.com/ and accepting the upgrade.
-          For more information, see the connector migration guide."
+        message: >
+          This version uses the DuckDB 0.9.1 database driver, which is not
+          backwards compatible with prior versions. MotherDuck users can
+          upgrade their database by visiting https://app.motherduck.com/ and
+          accepting the upgrade. For more information, see the connector
+          migration guide.
         upgradeDeadline: "2023-10-31"
+      0.4.0:
+        message: >
+          This version uses the DuckDB 0.10.3 database driver, which in not backwards compatible with databases created using the 0.9.x versions of DuckDB. MotherDuck users can upgrade their database by visiting https://app.motherduck.com/ and accepting the upgrade. For more information, see the connector migration guide.
+        upgradeDeadline: "2024-06-30"
   resourceRequirements:
     jobSpecific:
       - jobType: check_connection