master merge for 1.18.0 release

.github/workflows/build_docs.yml

@@ -0,0 +1,36 @@
+name: docs | build docs
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  build_docs:
+    name: docs | build docs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 9.13.2
+
+      - uses: actions/setup-node@v5
+        with:
+          node-version: '22'
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install node dependencies
+        run: cd docs/website && npm install
+
+      - name: Install python dependencies
+        run: cd docs/website && pip install -r requirements.txt
+
+      - name: Build docs
+        run: cd docs/website && npm run build:cloudflare

.github/workflows/main.yml

@@ -30,6 +30,11 @@ jobs:
   test_docs_snippets:
     name: test snippets in docs
     uses: ./.github/workflows/test_docs_snippets.yml
+
+  # NOTE: we build docs the same way as on cloudflare, so we can catch problems early
+  build_docs:
+    name: build docs
+    uses: ./.github/workflows/build_docs.yml
 
   lint:
     name: lint on all python versions
@@ -61,10 +66,10 @@ jobs:
     needs: test_common
     uses: ./.github/workflows/test_sources_local.yml
 
-  test_hub:
-    name: test dlthub features
-    needs: lint
-    uses: ./.github/workflows/test_hub.yml
+  # test_hub:
+  #   name: test dlthub features
+  #   needs: lint
+  #   uses: ./.github/workflows/test_hub.yml
 
   test_tools_airflow:
     name: test airflow helpers

.github/workflows/test_common.yml

@@ -43,9 +43,9 @@ jobs:
             python-version: "3.13"
             shell: bash
 
-          # - os: ubuntu-latest
-          #   python-version: "3.14.0-beta.4"
-          #   shell: bash
+          - os: "macos-14"
+            python-version: "3.14"
+            shell: bash
 
           # linux test with minimal dependencies
           - os: ubuntu-latest
@@ -63,11 +63,11 @@ jobs:
           - os: windows-latest
             python-version: "3.11"
             shell: cmd
-            pytest_args: '-m "not forked"'
+            pytest_args: '-m "not forked and not rfam"'
           - os: windows-latest
             python-version: "3.13"
             shell: cmd
-            pytest_args: '-m "not forked"'
+            pytest_args: '-m "not forked and not rfam"'
 
     defaults:
       run:
@@ -114,31 +114,40 @@ jobs:
 
       - name: Install pyarrow
         run: uv sync ${{ matrix.uv_sync_args }} --extra duckdb --extra cli --extra parquet --group sentry-sdk
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       - name: Run pipeline tests with pyarrow but no pandas installed
         run: |
           pytest tests/pipeline/test_pipeline_extra.py -k arrow ${{ matrix.pytest_args }}
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
+
+      - name: Install workspace dependencies
+        run: uv sync ${{ matrix.uv_sync_args }} --extra workspace --extra cli --group sentry-sdk
+        if: matrix.python-version != '3.14'
+
+      - name: Run workspace tests
+        run: |
+          pytest tests/workspace ${{ matrix.pytest_args }}
+        if: matrix.python-version != '3.14'
 
       - name: Install pipeline and sources dependencies
-        run: uv sync ${{ matrix.uv_sync_args }} --extra duckdb --extra cli --extra parquet --extra deltalake --extra sql_database --group sentry-sdk --group pipeline --group sources --group ibis
-        if: matrix.python-version != '3.14.0-beta.4'
+        run: uv sync ${{ matrix.uv_sync_args }} --extra http --extra duckdb --extra cli --extra parquet --extra deltalake --extra sql_database --group sentry-sdk --group pipeline --group sources --group ibis
+        if: matrix.python-version != '3.14'
 
       - name: Run extract and pipeline tests
         run: |
-          pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources ${{ matrix.pytest_args }}
-        if: matrix.python-version != '3.14.0-beta.4'
+          pytest tests/extract tests/pipeline tests/libs tests/destinations tests/sources ${{ matrix.pytest_args }}
+        if: matrix.python-version != '3.14'
 
       # here we upgrade sql alchemy to 2 an run the sql_database tests again
       - name: Upgrade sql alchemy
         run: uv run pip install sqlalchemy==2.0.32
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       - name: Run extract and pipeline tests 
         run: |
           pytest tests/sources/sql_database
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
   matrix_job_required_check:
     name: common | common tests
     needs: run_common

.github/workflows/test_destinations_local.yml

@@ -33,13 +33,11 @@ jobs:
             post_install_commands: "uv run pip install sqlalchemy==2.0.18" # minimum version required by `pyiceberg`
 
           # TODO: also test ducklake in remote mode with a buckets and remote postgres
-          - name: postgres, duckdb, ducklake, and dummy with cli commands
+          - name: postgres, duckdb, ducklake, and dummy
             destinations: "[\"postgres\", \"duckdb\", \"ducklake\", \"dummy\"]"
             filesystem_drivers: "[\"memory\", \"file\"]"
             extras: "--group adbc --extra postgres --extra postgis --extra parquet --extra duckdb --extra cli --extra filesystem"
             needs_postgres: true
-            additional_tests: "pytest tests/cli"
-
 
           # Clickhouse OSS (TODO: test with minio s3)
           - name: clickhouse

.github/workflows/test_docs_snippets.yml

@@ -71,8 +71,8 @@ jobs:
       - name: Install dependencies
         run: uv sync --extra duckdb --extra weaviate --extra parquet --extra qdrant --extra bigquery --extra postgres --extra lancedb  --extra s3 --extra workspace --group docs --group sentry-sdk --group ibis --group providers
 
-      - name: Install dlt-plus incl alpha releases
-        run: uv run pip install --pre dlt-plus
+      # - name: Install dlthub incl alpha releases
+      #   run: uv run pip install --pre dlthub
 
       - name: run docs preprocessor
         run: make preprocess-docs

.github/workflows/test_examples.yml

@@ -47,8 +47,8 @@ jobs:
       - name: Install dependencies
         run: uv sync --extra duckdb --extra weaviate --extra parquet --extra qdrant --extra bigquery --extra postgres --extra lancedb  --extra s3 --extra workspace --group docs --group sentry-sdk --group ibis --group providers
 
-      - name: Install dlt-plus incl alpha releases
-        run: uv run pip install --pre dlt-plus
+      # - name: Install dlthub incl alpha releases
+      #   run: uv run pip install --pre dlthub
 
       - name: create secrets.toml for examples
         run: pwd && echo "$DLT_SECRETS_TOML" > docs/examples/.dlt/secrets.toml

.github/workflows/test_hub.yml

@@ -1,7 +1,7 @@
 name: hub | dlthub features
 
 #
-# dlt-plus smoke tests against the nightly build.
+# dlthub smoke tests against the nightly build.
 #
 
 on:
@@ -19,7 +19,7 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         python-version: ["3.10", "3.11", "3.12"]
-        plus_dep: ["dlt-plus", "https://dlt-packages.fra1.digitaloceanspaces.com/dlt-plus/dlt_plus-0.0.0+nightly-py3-none-any.whl"]
+        dlthub_dep: ["dlthub", "https://dlt-packages.fra1.digitaloceanspaces.com/dlthub/dlthub-0.0.0+nightly-py3-none-any.whl"]
         # Test all python versions on ubuntu only
         exclude:
         - os: "macos-latest"
@@ -61,8 +61,8 @@ jobs:
       - name: Install all dependencies
         run: make dev
 
-      - name: Install dlt-plus nightly devel build without cache
-        run: uv run pip install --upgrade --force-reinstall --no-cache-dir ${{ matrix.plus_dep }}
+      - name: Install dlthub nightly devel build without cache
+        run: uv run pip install --upgrade --force-reinstall --no-cache-dir ${{ matrix.dlthub_dep }}
 
       - name: Run tests
         run: pytest tests/hub

.github/workflows/test_tools_dashboard.yml

@@ -102,18 +102,19 @@ jobs:
       # Run pipeline dashboard unit tests
       - name: Run pipeline dashboard unit tests
         run: |
-          pytest tests/helpers/dashboard
+          pytest tests/workspace/helpers/dashboard
 
-      # Run pipeline dashboard e2e tests (does not pass with python 3.9, does not pass on windows (playwright does not work somehow), does not pass on python 3.13 (ibis not available))
-      # Mac is also disabled for the time being
+      # Run pipeline dashboard e2e tests (does not pass with python 3.9
       - name: Run dashboard e2e
         run: |
-          marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
+          marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
 
+      # note that this test will pass only when running from cmd shell (_storage\.dlt\pipelines\ must stay)
       - name: Run dashboard e2e windows
         run: |
-          start marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
+          start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
+          timeout /t 6 /nobreak
           pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
 

Makefile

@@ -48,8 +48,10 @@ dev: has-uv
 
 dev-airflow: has-uv
 	uv sync --all-extras --group docs --group providers --group pipeline --group sources --group sentry-sdk --group ibis --group airflow
-
-lint:
+
+lint: lint-core lint-security lint-docstrings
+
+lint-core:
 	uv run mypy --config-file mypy.ini dlt tests
 	# NOTE: we need to make sure docstring_parser_fork is the only version of docstring_parser installed
 	uv pip uninstall docstring_parser
@@ -59,22 +61,20 @@ lint:
 	uv run flake8 --extend-ignore=D --max-line-length=200 dlt
 	uv run flake8 --extend-ignore=D --max-line-length=200 tests --exclude tests/reflection/module_cases,tests/common/reflection/cases/modules/
 	uv run black dlt docs tests --check --diff --color --extend-exclude=".*syntax_error.py"
-	$(MAKE) lint-security
-	$(MAKE) lint-docstrings
 
 format:
 	uv run black dlt docs tests --extend-exclude='.*syntax_error.py|_storage/.*'
 	uv run black docs/education --ipynb --extend-exclude='.*syntax_error.py|_storage/.*'
 
 lint-snippets:
 	cd docs/tools && uv run python check_embedded_snippets.py full
-
-lint-and-test-snippets: lint-snippets
-	# TODO: re-enable transformation snippets tests
+	# TODO: re-enable transformation snippets tests when dlthub dep is available
 	uv pip install docstring_parser_fork --reinstall
-	uv run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed --exclude docs/website/versioned_docs/ --exclude docs/website/docs/general-usage/transformations/transformation-snippets.py
+	uv run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed --exclude docs/website/versioned_docs/
 	uv run ruff check
-	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo --exclude docs/website/docs/hub/features/transformations/transformation-snippets.py
+	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo,docs/website/node_modules
+
+lint-and-test-snippets: lint-snippets
 	cd docs/website/docs && uv run pytest --ignore=node_modules --ignore hub/features/transformations/transformation-snippets.py
 
 lint-and-test-examples:
@@ -123,12 +123,7 @@ test-load-local-postgres:
 	DESTINATION__POSTGRES__CREDENTIALS=postgresql://loader:loader@localhost:5432/dlt_data ACTIVE_DESTINATIONS='["postgres"]' ALL_FILESYSTEM_DRIVERS='["memory"]'  uv run pytest tests/load
 
 test-common:
-	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations
-
-reset-test-storage:
-	-rm -r _storage
-	mkdir _storage
-	python3 tests/tools/create_storages.py
+	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/workspace tests/load/test_dummy_client.py tests/libs tests/destinations
 
 build-library: dev
 	uv version
@@ -180,8 +175,8 @@ test-e2e-dashboard-headed:
 	uv run pytest --headed --browser chromium tests/e2e
 
 start-dlt-dashboard-e2e:
-	uv run marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
+	uv run marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
 
 # creates the dashboard test pipelines globally for manual testing of the dashboard app and cli
 create-test-pipelines:
-	uv run python tests/helpers/dashboard/example_pipelines.py
+	uv run python tests/workspace/helpers/dashboard/example_pipelines.py

README.md

@@ -31,7 +31,7 @@ Be it a Google Colab notebook, AWS Lambda function, an Airflow DAG, your local l
 
 ## Installation
 
-dlt supports Python 3.9 through Python 3.14 (beta 4). Note that some optional extras are not yet available for Python 3.14, so support for this version is considered experimental.
+dlt supports Python 3.9 through Python 3.14. Note that some optional extras are not yet available for Python 3.14, so support for this version is considered experimental.
 
 ```sh
 pip install dlt

dlt/__init__.py

@@ -73,6 +73,7 @@
     "attach",
     "Pipeline",
     "dbt",
+    "hub",
     "progress",
     "current",
     "mark",

dlt/__main__.py

@@ -1,4 +1,4 @@
-from dlt.cli._dlt import main
+from dlt._workspace.cli._dlt import main
 
 if __name__ == "__main__":
     main()

dlt/__plugins__.py

@@ -0,0 +1,18 @@
+"""A module that imports all pluggy plugins implementations to work with pyproject entrypoint"""
+from typing import Any, Dict, Optional
+
+from dlt.common.configuration import plugins as _plugins
+from dlt.common.configuration.specs.pluggable_run_context import RunContextBase
+from dlt.common.runtime.run_context import RunContext
+
+# import workspace plugins before core plugin
+from dlt._workspace._plugins import *  # noqa
+from dlt._workspace.cli._plugins import *  # noqa
+
+
+# make sure OSS context is tried last
+@_plugins.hookimpl(specname="plug_run_context", trylast=True)
+def plug_run_context_impl(
+    run_dir: Optional[str], runtime_kwargs: Optional[Dict[str, Any]]
+) -> Optional[RunContextBase]:
+    return RunContext(run_dir)

dlt/_workspace/_plugins.py

@@ -0,0 +1,35 @@
+import os
+from typing import Any, Dict, Optional
+
+from dlt._workspace.exceptions import WorkspaceRunContextNotAvailable
+from dlt._workspace.profile import DEFAULT_PROFILE, read_profile_pin
+from dlt.common.configuration import plugins as _plugins
+from dlt.common.configuration.specs.pluggable_run_context import RunContextBase
+from dlt.common.runtime.run_context import RunContext
+
+from dlt._workspace.run_context import default_name
+from dlt._workspace._workspace_context import (
+    WorkspaceRunContext,
+    is_workspace_dir,
+)
+
+__all__ = ["plug_workspace_context_impl"]
+
+
+@_plugins.hookimpl(specname="plug_run_context", trylast=False)
+def plug_workspace_context_impl(
+    run_dir: Optional[str], runtime_kwargs: Optional[Dict[str, Any]]
+) -> Optional[RunContextBase]:
+    # TODO: if recursive search was requested
+    # if runtime_kwargs.get("_look_recursive")
+    run_dir = os.path.abspath(run_dir or ".")
+    if is_workspace_dir(run_dir):
+        profile: str = None
+        if runtime_kwargs:
+            profile = runtime_kwargs.get("profile")
+        profile = profile or read_profile_pin(RunContext(run_dir)) or DEFAULT_PROFILE
+        return WorkspaceRunContext(default_name(run_dir), run_dir, profile)
+    elif runtime_kwargs and runtime_kwargs.get("_required") == "WorkspaceRunContext":
+        raise WorkspaceRunContextNotAvailable(run_dir)
+
+    return None