Code simplified

Author: anuunchin

dlt/common/destination/client.py

@@ -142,11 +142,6 @@ def from_normalized_mapping(
         )
 
 
-@configspec
-class DestinationTypeConfiguration(BaseConfiguration):
-    destination_type: str = None
-
-
 @configspec
 class DestinationClientConfiguration(BaseConfiguration):
     destination_type: Annotated[str, NotResolved()] = dataclasses.field(

dlt/common/destination/exceptions.py

@@ -1,4 +1,5 @@
 from typing import Any, Iterable, List, Sequence, Optional
+import textwrap
 
 from dlt.common.exceptions import DltException, TerminalException, TransientException
 from dlt.common.reflection.exceptions import ReferenceImportError
@@ -15,20 +16,53 @@ def __init__(
         ref: str,
         qualified_refs: Sequence[str],
         traces: Sequence[ImportTrace],
-        from_name: bool = False,
+        destination_type: str = None,
+        named_dest_attempted: bool = False,
     ) -> None:
         self.ref = ref
         self.qualified_refs = qualified_refs
-        self.from_name = from_name
+        self.destination_type = destination_type
+        self.named_dest_attempted = named_dest_attempted
         super().__init__(traces=traces)
 
     def __str__(self) -> str:
+        msg = ""
         if "." in self.ref:
-            msg = f"Destination module `{self.ref}` is not registered."
+            msg += f"Destination module `{self.ref}` is not registered."
         else:
-            msg = (
-                f"Destination{' type' if self.from_name else ''} `{self.ref}` is not one of the"
-                f" standard dlt destination{' types' if self.from_name else 's'}."
+            if self.named_dest_attempted:
+                msg += (
+                    f"Destination '{self.ref}' was first attempted to be resolved as a named"
+                    " destination with a configured type. "
+                )
+                if self.destination_type:
+                    msg += (
+                        f"However, the configured destination type '{self.destination_type}' is not"
+                        " valid. Set a valid destination type. "
+                    )
+                else:
+                    msg += (
+                        "However, no destination type was configured. "
+                        "If your destination is a named destination, "
+                        "set a valid destination type either as an environment variable:\n\n"
+                    )
+                    msg += textwrap.indent(
+                        f"DESTINATION__{self.ref.upper()}__DESTINATION_TYPE=duckdb\n", "  "
+                    )
+                    msg += "\nor in your configuration files:\n\n"
+                    msg += textwrap.indent(
+                        f'[destination.{self.ref}]\ndestination_type="duckdb"\n\n', "  "
+                    )
+
+                msg += (
+                    f"Since no{' valid' if self.destination_type else ''} destination type was"
+                    f" found, dlt also tried to resolve '{self.ref}' as a standard destination."
+                    " However, "
+                )
+
+            msg += (
+                f"{'d' if self.named_dest_attempted else 'D'}estination `{self.ref}` is not one of"
+                " the standard dlt destinations."
             )
 
         if len(self.qualified_refs) == 1 and self.qualified_refs[0] == self.ref:
@@ -43,31 +77,6 @@ def __str__(self) -> str:
         return msg
 
 
-class DestinationTypeResolutionException(DestinationException):
-    def __init__(
-        self,
-        ref: str,
-        type_resolution_error: Exception,
-        named_dest_error: Optional[Exception],
-    ) -> None:
-        self.ref = ref
-        self.named_dest_error = named_dest_error
-        self.type_resolution_error = type_resolution_error
-
-        msg = f"Failed to resolve destination '{ref}'"
-
-        if named_dest_error:
-            msg += (
-                ". First tried to resolve as a named destination with destination type, "
-                f"but failed: {named_dest_error}. "
-                f"Then tried to resolve as destination type, but failed: {type_resolution_error}"
-            )
-        else:
-            msg += f" as destination type: {type_resolution_error}"
-
-        super().__init__(msg)
-
-
 class InvalidDestinationReference(DestinationException):
     def __init__(self, refs: Any) -> None:
         self.refs = refs

dlt/common/destination/reference.py

@@ -16,21 +16,17 @@
 from typing_extensions import TypeAlias
 import inspect
 
+import dlt
 from dlt.common import logger
 from dlt.common.configuration.specs.base_configuration import BaseConfiguration
 from dlt.common.normalizers.naming import NamingConvention
 from dlt.common.configuration import resolve_configuration, known_sections
 from dlt.common.destination.capabilities import DestinationCapabilitiesContext
 from dlt.common.destination.exceptions import (
-    DestinationTypeResolutionException,
     InvalidDestinationReference,
     UnknownDestinationModule,
 )
-from dlt.common.destination.client import (
-    DestinationClientConfiguration,
-    JobClientBase,
-    DestinationTypeConfiguration,
-)
+from dlt.common.destination.client import DestinationClientConfiguration, JobClientBase
 from dlt.common.runtime.run_context import get_plugin_modules
 from dlt.common.schema.schema import Schema
 from dlt.common.typing import is_subclass
@@ -272,37 +268,40 @@ def from_reference(
                 )
             return ref
 
-        # If destination_name is not provided and ref does not contain dots
-        # try to resolve named destination with destination type
-        named_dest_error = None
-        if not destination_name and "." not in ref:
+        # If destination name is provided or ref is a module ref
+        # don't attempt to resolve as named destination
+        if destination_name or "." in ref:
+            return DestinationReference.from_reference(
+                ref, credentials, destination_name, environment, **kwargs
+            )
+
+        # First, try to resolve as a named destination with configured type
+        destination_type: str = dlt.config.get(f"destination.{ref}.destination_type")
+        if destination_type:
             try:
                 return DestinationReference.from_name(
-                    credentials=credentials,
+                    destination_type=destination_type,
                     destination_name=ref,
+                    credentials=credentials,
                     environment=environment,
                     **kwargs,
                 )
-            except Exception as e:
-                named_dest_error = e
+            except Exception:
+                pass
 
-        # Fallback to shorthand type reference
+        # Then, try to resolve as a shorthand destination ref
         try:
-            dest_ref = DestinationReference.from_reference(
+            return DestinationReference.from_reference(
                 ref, credentials, destination_name, environment, **kwargs
             )
-            return dest_ref
-        except Exception as e:
-            if named_dest_error is None:
-                # Only direct type resolution was attempted, raise original error
-                raise e
-            else:
-                # Both resolution methods failed, use comprehensive exception
-                raise DestinationTypeResolutionException(
-                    ref=ref,
-                    type_resolution_error=e,
-                    named_dest_error=named_dest_error,
-                )
+        except UnknownDestinationModule as e:
+            raise UnknownDestinationModule(
+                ref=e.ref,
+                qualified_refs=e.qualified_refs,
+                traces=e.traces,
+                destination_type=destination_type,
+                named_dest_attempted=True,
+            ) from e.__cause__
 
 
 class DestinationReference:
@@ -415,31 +414,23 @@ def from_reference(
     @classmethod
     def from_name(
         cls,
+        destination_type: str,
         destination_name: str,
         credentials: Optional[Any] = None,
         environment: Optional[str] = None,
         **kwargs: Any,
     ) -> Optional[AnyDestination]:
-        resolved_config = resolve_configuration(
-            DestinationTypeConfiguration(),
-            sections=(known_sections.DESTINATION, destination_name),
+        """Instantiate destination from a destination type and name.
+        This method creates a named destination by using the destination type
+        as a factory and instantiating it with the provided destination name.
+        """
+        return DestinationReference.from_reference(
+            ref=destination_type,
+            credentials=credentials,
+            destination_name=destination_name,
+            environment=environment,
+            **kwargs,
         )
-        destination_type = resolved_config.destination_type
-        try:
-            return DestinationReference.from_reference(
-                ref=destination_type,
-                credentials=credentials,
-                destination_name=destination_name,
-                environment=environment,
-                **kwargs,
-            )
-        except UnknownDestinationModule as e:
-            raise UnknownDestinationModule(
-                ref=e.ref,
-                qualified_refs=e.qualified_refs,
-                traces=e.traces,
-                from_name=True,
-            )
 
     @staticmethod
     def normalize_type(destination_type: str) -> str:

tests/common/destination/test_reference.py

@@ -8,10 +8,7 @@
 from dlt.common.destination import Destination, DestinationReference
 from dlt.common.destination.client import DestinationClientDwhConfiguration, WithStagingDataset
 from dlt.common.destination import DestinationCapabilitiesContext
-from dlt.common.destination.exceptions import (
-    UnknownDestinationModule,
-    DestinationTypeResolutionException,
-)
+from dlt.common.destination.exceptions import UnknownDestinationModule
 from dlt.common.schema import Schema
 from dlt.common.typing import is_subclass
 from dlt.common.normalizers.naming import sql_ci_v1, sql_cs_v1
@@ -28,10 +25,9 @@ def test_import_unknown_destination() -> None:
     assert unk_ex.value.qualified_refs == [
         "meltdb",
         "dlt.destinations.meltdb",
-        "dlt.destinations.meltdb",
     ]
     traces = unk_ex.value.traces
-    assert len(traces) == 2 and traces[0].reason == "AttrNotFound"
+    assert len(traces) == 1 and traces[0].reason == "AttrNotFound"
 
     # custom module
     with pytest.raises(UnknownDestinationModule) as unk_ex:
@@ -331,43 +327,44 @@ def test_import_destination_type_config(
     if destination_type == "wrong_type":
         # Case 1: Fully qualified ref with dots
         # Skips named destination resolution and only attempts direct type resolution
-        with pytest.raises(UnknownDestinationModule) as module_ex:
+        with pytest.raises(UnknownDestinationModule) as py_exc:
             Destination.from_reference(ref=f"dlt.destinations.{destination_type}")
-        assert "`dlt.destinations.wrong_type` is not registered." in str(module_ex.value)
+        assert "`dlt.destinations.wrong_type` is not registered" in str(py_exc.value)
+        assert not py_exc.value.named_dest_attempted
+        assert not py_exc.value.destination_type
 
         # Case 2: Explicit destination_name provided
         # Same as Case 1
-        with pytest.raises(UnknownDestinationModule) as module_ex:
+        with pytest.raises(UnknownDestinationModule) as py_exc:
             Destination.from_reference(ref=destination_type, destination_name="my_destination")
-        assert "`wrong_type` is not one of the standard dlt destinations." in str(module_ex.value)
+        assert "`wrong_type` is not one of the standard dlt destinations" in str(py_exc.value)
+        assert not py_exc.value.named_dest_attempted
+        assert not py_exc.value.destination_type
 
         # Case 3: Named destination with invalid configured type
         # First tries named destination "my_destination" with configured type "wrong_type"
         # Then tries "my_destination" as destination type
-        with pytest.raises(DestinationTypeResolutionException) as resolution_exc:
+        with pytest.raises(UnknownDestinationModule) as py_exc:
             Destination.from_reference(ref="my_destination")
-        assert resolution_exc.value.named_dest_error
-        assert "`wrong_type` is not one of the standard dlt destination types." in str(
-            resolution_exc.value
-        )
-        assert "`my_destination` is not one of the standard dlt destinations." in str(
-            resolution_exc.value
+        assert f"destination type '{destination_type}' is not valid" in str(py_exc.value)
+        assert "dlt also tried to resolve 'my_destination' as a standard destination" in str(
+            py_exc.value
         )
+        assert py_exc.value.named_dest_attempted is True
+        assert py_exc.value.destination_type == "wrong_type"
 
         # Case 4: Named destination with missing type configuration
         # First tries named destination "my_destination" but no type configured (config error)
         # Then tries "my_destination" as direct destination type
         environment.clear()
-        with pytest.raises(DestinationTypeResolutionException) as resolution_exc:
+        with pytest.raises(UnknownDestinationModule) as py_exc:
             Destination.from_reference(ref="my_destination")
-        assert resolution_exc.value.named_dest_error
-        assert (
-            "Missing 1 field(s) in configuration `DestinationTypeConfiguration`: `destination_type`"
-            in str(resolution_exc.value)
-        )
-        assert "`my_destination` is not one of the standard dlt destinations." in str(
-            resolution_exc.value
+        assert "no destination type was configured" in str(py_exc.value)
+        assert "dlt also tried to resolve 'my_destination' as a standard destination" in str(
+            py_exc.value
         )
+        assert py_exc.value.named_dest_attempted is True
+        assert not py_exc.value.destination_type
 
     else:
         dest = Destination.from_reference(ref="my_destination")

tests/common/runtime/test_run_context.py

@@ -65,7 +65,7 @@ def test_run_context() -> None:
 
     # check plugin modules
     # NOTE: first `dlt` - is the root module of current context, second is always present
-    assert get_plugin_modules() == ["dlt", "dlt"]
+    assert get_plugin_modules() == ["dlt"]
 
 
 def test_context_without_module() -> None:

tests/destinations/test_custom_destination.py

@@ -11,17 +11,14 @@
 from dlt.common.schema import TTableSchema
 from dlt.common.data_writers.writers import TLoaderFileFormat
 from dlt.common.destination import Destination, DestinationReference
-from dlt.common.destination.exceptions import (
-    DestinationTransientException,
-    DestinationTypeResolutionException,
-)
+from dlt.common.destination.exceptions import DestinationTransientException
 from dlt.common.configuration.exceptions import ConfigFieldMissingException, ConfigurationValueError
 from dlt.common.configuration.specs import ConnectionStringCredentials
 from dlt.common.configuration.inject import get_fun_spec
 from dlt.common.configuration.specs import BaseConfiguration
 
 from dlt.destinations.impl.destination.configuration import CustomDestinationClientConfiguration
-from dlt.destinations.impl.destination.factory import destination
+from dlt.destinations.impl.destination.factory import UnknownCustomDestinationCallable, destination
 from dlt.pipeline.exceptions import PipelineStepFailed
 
 from tests.cases import table_update_and_row
@@ -293,7 +290,7 @@ def local_sink_func_no_params(items: TDataItems, table: TTableSchema) -> None:
         p.run([1, 2, 3], table_name="items")
 
     # pass invalid string reference will fail on instantiation
-    with pytest.raises(DestinationTypeResolutionException):
+    with pytest.raises(UnknownCustomDestinationCallable):
         p = dlt.pipeline(
             "sink_test",
             destination=Destination.from_reference(

tests/load/pipeline/test_pipelines.py

@@ -10,10 +10,7 @@
 from dlt.common.pipeline import SupportsPipeline
 from dlt.common.destination import Destination
 from dlt.common.destination.client import WithStagingDataset
-from dlt.common.destination.exceptions import (
-    UnknownDestinationModule,
-    DestinationTypeResolutionException,
-)
+from dlt.common.destination.exceptions import UnknownDestinationModule
 from dlt.common.schema.schema import Schema
 from dlt.common.schema.typing import VERSION_TABLE_NAME, REPLACE_STRATEGIES, TLoaderReplaceStrategy
 from dlt.common.schema.utils import new_table
@@ -1149,14 +1146,14 @@ def test_data():
     assert_load_info(info)
 
     # test unconfigured destination name
-    with pytest.raises(DestinationTypeResolutionException) as py_exc:
+    with pytest.raises(UnknownDestinationModule) as py_exc:
         dlt.pipeline(destination="another_custom_name")
-    assert py_exc.value.named_dest_error
-    assert (
-        "Missing 1 field(s) in configuration `DestinationTypeConfiguration`: `destination_type`"
-        in str(py_exc.value)
-    )
+    assert py_exc.value.named_dest_attempted is True
+    assert not py_exc.value.destination_type
+    assert "no destination type was configured" in str(py_exc.value)
 
     # if destination contains dots, no fallbacks must happen
-    with pytest.raises(UnknownDestinationModule):
+    with pytest.raises(UnknownDestinationModule) as py_exc:
         dlt.pipeline(destination="dlt.destinations.unknown")
+    assert not py_exc.value.named_dest_attempted
+    assert not py_exc.value.destination_type