Tests fixed, exception improved, docs adjusted

Author: anuunchin

dlt/common/destination/exceptions.py

@@ -1,4 +1,4 @@
-from typing import Any, Iterable, List, Sequence
+from typing import Any, Iterable, List, Sequence, Optional
 
 from dlt.common.exceptions import DltException, TerminalException, TransientException
 from dlt.common.reflection.exceptions import ReferenceImportError
@@ -43,6 +43,31 @@ def __str__(self) -> str:
         return msg
 
 
+class DestinationTypeResolutionException(DestinationException):
+    def __init__(
+        self,
+        ref: str,
+        type_resolution_error: Exception,
+        named_dest_error: Optional[Exception],
+    ) -> None:
+        self.ref = ref
+        self.named_dest_error = named_dest_error
+        self.type_resolution_error = type_resolution_error
+
+        msg = f"Failed to resolve destination '{ref}'"
+
+        if named_dest_error:
+            msg += (
+                ". First tried to resolve as a named destination with destination type, "
+                f"but failed: {named_dest_error}. "
+                f"Then tried to resolve as destination type, but failed: {type_resolution_error}"
+            )
+        else:
+            msg += f" as destination type: {type_resolution_error}"
+
+        super().__init__(msg)
+
+
 class InvalidDestinationReference(DestinationException):
     def __init__(self, refs: Any) -> None:
         self.refs = refs

dlt/common/destination/reference.py

@@ -22,7 +22,7 @@
 from dlt.common.configuration import resolve_configuration, known_sections
 from dlt.common.destination.capabilities import DestinationCapabilitiesContext
 from dlt.common.destination.exceptions import (
-    DestinationException,
+    DestinationTypeResolutionException,
     InvalidDestinationReference,
     UnknownDestinationModule,
 )
@@ -285,27 +285,24 @@ def from_reference(
                 )
             except Exception as e:
                 named_dest_error = e
-                logger.debug(
-                    f"Tried to resolve destination '{ref}' as a named destination with destination"
-                    f" type, but got the following error: {e}"
-                    "Will try to resolve destination as destination type."
-                )
 
         # Fallback to shorthand type reference
         try:
             dest_ref = DestinationReference.from_reference(
                 ref, credentials, destination_name, environment, **kwargs
             )
-            logger.info(f"Resolved destination '{ref}' as destination of type '{ref}'.")
             return dest_ref
         except Exception as e:
-            if named_dest_error:
-                logger.error(
-                    "First tried to resolve destination as a named destination with destination"
-                    f" type, but failed: {named_dest_error}Then tried to resolve destination as"
-                    f" destination type, but failed: {e}"
+            if named_dest_error is None:
+                # Only direct type resolution was attempted, raise original error
+                raise e
+            else:
+                # Both resolution methods failed, use comprehensive exception
+                raise DestinationTypeResolutionException(
+                    ref=ref,
+                    type_resolution_error=e,
+                    named_dest_error=named_dest_error,
                 )
-            raise
 
 
 class DestinationReference:

docs/website/docs/general-usage/destination.md

@@ -18,6 +18,21 @@ We recommend that you declare the destination type when creating a pipeline inst
 
 Above, we want to use the **filesystem** built-in destination. You can use shorthand types only for built-ins.
 
+* Use a **custom destination name** with a configured type
+<!--@@@DLT_SNIPPET ./snippets/destination-snippets.py::custom_destination_name-->
+
+Above, we use a custom destination name and configure the destination type to **filesystem** using an environment variable.
+
+:::note
+When resolving non-module destination references (e.g., `"filesystem"` or `"my_destination"`, not `"dlt.destinations.filesystem"`), dlt first attempts to resolve the reference as a named destination with a valid destination type configured, then falls back to shorthand type resolution.
+
+This means that, in the previous example, if the destination type is not properly configured or is not a valid destination type, dlt will attempt to resolve `"my_destination"` as a shorthand for a built-in type and will eventually fail.
+
+As another example, the following:
+<!--@@@DLT_SNIPPET ./snippets/destination-snippets.py::avoid_example-->
+will be resolved as a BigQuery destination that is named `"filesystem"`!
+:::
+
 * Use full **destination factory type**
 <!--@@@DLT_SNIPPET ./snippets/destination-snippets.py::class_type-->
 
@@ -30,32 +45,34 @@ Above, we import the destination factory for **filesystem** and pass it to the p
 
 All examples above will create the same destination class with default parameters and pull required config and secret values from [configuration](credentials/index.md) - they are equivalent.
 
-
-### Pass explicit parameters and a name to a destination
+### Pass explicit parameters and a name to a destination factory
 You can instantiate the **destination factory** yourself to configure it explicitly. When doing this, you work with destinations the same way you work with [sources](source.md)
 <!--@@@DLT_SNIPPET ./snippets/destination-snippets.py::instance-->
 
 Above, we import and instantiate the `filesystem` destination factory. We pass the explicit URL of the bucket and name the destination `production_az_bucket`.
 
-If a destination is not named, its shorthand type (the Python factory name) serves as a destination name. Name your destination explicitly if you need several separate configurations of destinations of the same type (i.e., you wish to maintain credentials for development, staging, and production storage buckets in the same config file). The destination name is also stored in the [load info](../running-in-production/running.md#inspect-and-save-the-load-info-and-trace) and pipeline traces, so use them also when you need more descriptive names (other than, for example, `filesystem`).
+If a destination is not named, its shorthand type (the Python factory name) serves as the destination name. Name your destination explicitly if you need several separate configurations for destinations of the same type (i.e., when you wish to maintain credentials for development, staging, and production storage buckets in the same config file). The destination name is also stored in the [load info](../running-in-production/running.md#inspect-and-save-the-load-info-and-trace) and pipeline traces, so use explicit names when you need more descriptive identifiers (rather than generic names like `filesystem`).
 
 
 ## Configure a destination
 We recommend passing the credentials and other required parameters to configuration via TOML files, environment variables, or other [config providers](credentials/setup). This allows you, for example, to easily switch to production destinations after deployment.
 
-We recommend using the [default config section layout](credentials/advanced#organize-configuration-and-secrets-with-sections) as below:
+Use the [default config section layout](credentials/advanced#organize-configuration-and-secrets-with-sections) as shown below:
 <!--@@@DLT_SNIPPET ./snippets/destination-toml.toml::default_layout-->
 
-or via environment variables:
+Alternatively, you can use environment variables:
 ```sh
 DESTINATION__FILESYSTEM__BUCKET_URL=az://dlt-azure-bucket
 DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_NAME=dltdata
 DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_KEY="storage key"
 ```
 
-For named destinations, you use their names in the config section
+When using named destination factories, use the destination name in the config section:
 <!--@@@DLT_SNIPPET ./snippets/destination-toml.toml::name_layout-->
 
+For custom destination names passed to your pipeline (e.g., `destination="my_destination"`), dlt resolves the destination type from configuration. Add `destination_type` to specify which destination type to use:
+<!--@@@DLT_SNIPPET ./snippets/destination-toml.toml::custom_name_layout-->
+
 
 Note that when you use the [`dlt init` command](../walkthroughs/add-a-verified-source.md) to create or add a data source, `dlt` creates a sample configuration for the selected destination.
 

docs/website/docs/general-usage/pipeline.md

@@ -31,7 +31,7 @@ You instantiate a pipeline by calling the `dlt.pipeline` function with the follo
   events and to restore its state and data schemas on subsequent runs. If not provided, `dlt` will
   create a pipeline name from the file name of the currently executing Python module.
 - `destination`: a name of the [destination](../dlt-ecosystem/destinations) to which dlt
-  will load the data. It may also be provided to the `run` method of the `pipeline`.
+  will load the data. It may also be provided to the `run` method of the `pipeline` and can be declared in [various ways](destination.md). 
 - `dataset_name`: a name of the dataset to which the data will be loaded. A dataset is a logical
   group of tables, i.e., `schema` in relational databases or a folder grouping many files. It may also be
   provided later to the `run` or `load` methods of the pipeline. If not provided, then
@@ -67,31 +67,6 @@ info = pipeline.run(generate_rows(10))
 
 print(info)
 ```
-## Named destination configuration
-You can use a custom destination name in your pipeline and configure the actual destination type through configuration files or environment variables. This approach allows you to switch between different destinations (e.g., from development to production) without changing your pipeline code.
-
-Example:
-```py
-import dlt
-
-pipeline = dlt.pipeline(destination="custom_pipeline", dataset_name="sql_database_data")
-```
-
-Then configure the actual destination type in your `.dlt/secrets.toml`:
-
-```toml
-[destination.custom_pipeline]
-# for development
-destination_type = "duckdb" 
-
-# for production
-# destination_type = "bigquery"
-
-# [destination.custom_pipeline.credentials]
-# project_id = "project_id" # please set me up!
-# private_key = "private_key" # please set me up!
-# client_email = "client_email" # please set me up!
-```
 
 ## Pipeline working directory
 

docs/website/docs/general-usage/snippets/destination-snippets.py

@@ -30,6 +30,30 @@ def destination_instantiation_snippet() -> None:
 
     assert pipeline.destination.destination_name == "filesystem"
 
+    # @@@DLT_SNIPPET_START custom_destination_name
+    import os
+    import dlt
+
+    os.environ["DESTINATION__MY_DESTINATION__DESTINATION_TYPE"] = "filesystem"
+
+    pipeline = dlt.pipeline("pipeline", destination="my_destination")
+    # @@@DLT_SNIPPET_END custom_destination_name
+
+    assert pipeline.destination.destination_type == "dlt.destinations.filesystem"
+    assert pipeline.destination.destination_name == "my_destination"
+
+    # @@@DLT_SNIPPET_START avoid_example
+    import os
+    import dlt
+
+    os.environ["DESTINATION__FILESYSTEM__DESTINATION_TYPE"] = "bigquery"
+
+    pipeline = dlt.pipeline("pipeline", destination="filesystem")
+    # @@@DLT_SNIPPET_END avoid_example
+
+    assert pipeline.destination.destination_type == "dlt.destinations.bigquery"
+    assert pipeline.destination.destination_name == "filesystem"
+
     # @@@DLT_SNIPPET_START instance
     import dlt
 

docs/website/docs/general-usage/snippets/destination-toml.toml

@@ -12,4 +12,13 @@ bucket_url="az://dlt-azure-bucket"
 [destination.production_az_bucket.credentials]
 azure_storage_account_name="dltdata"
 azure_storage_account_key="storage key"
-# @@@DLT_SNIPPET_END name_layout
+# @@@DLT_SNIPPET_END name_layout
+
+# @@@DLT_SNIPPET_START custom_name_layout
+[destination.my_destination]
+destination_type="filesystem"
+bucket_url="az://dlt-azure-bucket"
+[destination.my_destination.credentials]
+azure_storage_account_name="dltdata"
+azure_storage_account_key="storage key"
+# @@@DLT_SNIPPET_END custom_name_layout

tests/common/destination/test_reference.py

@@ -8,7 +8,10 @@
 from dlt.common.destination import Destination, DestinationReference
 from dlt.common.destination.client import DestinationClientDwhConfiguration, WithStagingDataset
 from dlt.common.destination import DestinationCapabilitiesContext
-from dlt.common.destination.exceptions import UnknownDestinationModule
+from dlt.common.destination.exceptions import (
+    UnknownDestinationModule,
+    DestinationTypeResolutionException,
+)
 from dlt.common.schema import Schema
 from dlt.common.typing import is_subclass
 from dlt.common.normalizers.naming import sql_ci_v1, sql_cs_v1
@@ -316,39 +319,66 @@ def test_import_destination_config() -> None:
 def test_import_destination_type_config(
     environment: Dict[str, str],
     destination_type: str,
-    destination_name: str = "my_destination",
 ) -> None:
-    environment[f"DESTINATION__{destination_name.upper()}__DESTINATION_TYPE"] = destination_type
-    msg_from_fallback = "configure a valid dlt destination type"
+    """Test destination resolution behavior with both valid and invalid destination types.
+
+    This test covers the resolution strategy where dlt first tries to resolve
+    a destination as a named destination with configured type, and if that fails,
+    falls back to resolving it as a direct destination type reference.
+    """
+    environment["DESTINATION__MY_DESTINATION__DESTINATION_TYPE"] = destination_type
+
     if destination_type == "wrong_type":
-        with pytest.raises(UnknownDestinationModule) as py_exc:
-            Destination.from_reference(ref=destination_name)
-        test = py_exc.value
-        assert msg_from_fallback in str(py_exc.value)
-
-        # if destination_name is provided, ref should be a valid destination type
-        with pytest.raises(UnknownDestinationModule) as py_exc:
-            Destination.from_reference(
-                ref=f"dlt.destinations.{destination_type}", destination_name=destination_name
-            )
-        assert msg_from_fallback not in str(py_exc.value)
-
-        # if ref contains dots, it's a module path and should be valid
-        # names with dots will not resolve correctly from configs anyway
-        with pytest.raises(UnknownDestinationModule):
+        # Case 1: Fully qualified ref with dots
+        # Skips named destination resolution and only attempts direct type resolution
+        with pytest.raises(UnknownDestinationModule) as module_ex:
             Destination.from_reference(ref=f"dlt.destinations.{destination_type}")
-        assert msg_from_fallback not in str(py_exc.value)
+        assert "`dlt.destinations.wrong_type` is not registered." in str(module_ex.value)
+
+        # Case 2: Explicit destination_name provided
+        # Same as Case 1
+        with pytest.raises(UnknownDestinationModule) as module_ex:
+            Destination.from_reference(ref=destination_type, destination_name="my_destination")
+        assert "`wrong_type` is not one of the standard dlt destinations." in str(module_ex.value)
+
+        # Case 3: Named destination with invalid configured type
+        # First tries named destination "my_destination" with configured type "wrong_type"
+        # Then tries "my_destination" as destination type
+        with pytest.raises(DestinationTypeResolutionException) as resolution_exc:
+            Destination.from_reference(ref="my_destination")
+        assert resolution_exc.value.named_dest_error
+        assert "`wrong_type` is not one of the standard dlt destination types." in str(
+            resolution_exc.value
+        )
+        assert "`my_destination` is not one of the standard dlt destinations." in str(
+            resolution_exc.value
+        )
+
+        # Case 4: Named destination with missing type configuration
+        # First tries named destination "my_destination" but no type configured (config error)
+        # Then tries "my_destination" as direct destination type
+        environment.clear()
+        with pytest.raises(DestinationTypeResolutionException) as resolution_exc:
+            Destination.from_reference(ref="my_destination")
+        assert resolution_exc.value.named_dest_error
+        assert (
+            "Missing 1 field(s) in configuration `DestinationTypeConfiguration`: `destination_type`"
+            in str(resolution_exc.value)
+        )
+        assert "`my_destination` is not one of the standard dlt destinations." in str(
+            resolution_exc.value
+        )
 
     else:
-        dest = Destination.from_reference(ref=destination_name)
+        dest = Destination.from_reference(ref="my_destination")
         assert dest.destination_type == "dlt.destinations.duckdb"
-        assert dest.destination_name == destination_name
+        assert dest.destination_name == "my_destination"
 
         dest = Destination.from_reference(
-            ref=f"dlt.destinations.{destination_type}", destination_name=destination_name
+            ref=f"dlt.destinations.{destination_type}", destination_name="my_destination"
         )
         assert dest.destination_type == "dlt.destinations.duckdb"
-        assert dest.destination_name == destination_name
+        assert dest.destination_name == "my_destination"
 
         dest = Destination.from_reference(ref=f"dlt.destinations.{destination_type}")
         assert dest.destination_type == "dlt.destinations.duckdb"

tests/destinations/test_custom_destination.py

@@ -11,14 +11,17 @@
 from dlt.common.schema import TTableSchema
 from dlt.common.data_writers.writers import TLoaderFileFormat
 from dlt.common.destination import Destination, DestinationReference
-from dlt.common.destination.exceptions import DestinationTransientException
+from dlt.common.destination.exceptions import (
+    DestinationTransientException,
+    DestinationTypeResolutionException,
+)
 from dlt.common.configuration.exceptions import ConfigFieldMissingException, ConfigurationValueError
 from dlt.common.configuration.specs import ConnectionStringCredentials
 from dlt.common.configuration.inject import get_fun_spec
 from dlt.common.configuration.specs import BaseConfiguration
 
 from dlt.destinations.impl.destination.configuration import CustomDestinationClientConfiguration
-from dlt.destinations.impl.destination.factory import UnknownCustomDestinationCallable, destination
+from dlt.destinations.impl.destination.factory import destination
 from dlt.pipeline.exceptions import PipelineStepFailed
 
 from tests.cases import table_update_and_row
@@ -290,7 +293,7 @@ def local_sink_func_no_params(items: TDataItems, table: TTableSchema) -> None:
         p.run([1, 2, 3], table_name="items")
 
     # pass invalid string reference will fail on instantiation
-    with pytest.raises(UnknownCustomDestinationCallable):
+    with pytest.raises(DestinationTypeResolutionException):
         p = dlt.pipeline(
             "sink_test",
             destination=Destination.from_reference(