Merge remote-tracking branch 'origin/devel' into docs/dlthub/init

Author: AstrakhantsevaAA

.github/workflows/test_common.yml

@@ -43,9 +43,9 @@ jobs:
             python-version: "3.13"
             shell: bash
 
-          # - os: ubuntu-latest
-          #   python-version: "3.14.0-beta.4"
-          #   shell: bash
+          - os: "macos-14"
+            python-version: "3.14"
+            shell: bash
 
           # linux test with minimal dependencies
           - os: ubuntu-latest
@@ -114,40 +114,40 @@ jobs:
 
       - name: Install pyarrow
         run: uv sync ${{ matrix.uv_sync_args }} --extra duckdb --extra cli --extra parquet --group sentry-sdk
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       - name: Run pipeline tests with pyarrow but no pandas installed
         run: |
           pytest tests/pipeline/test_pipeline_extra.py -k arrow ${{ matrix.pytest_args }}
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       - name: Install workspace dependencies
         run: uv sync ${{ matrix.uv_sync_args }} --extra workspace --extra cli --group sentry-sdk
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       - name: Run workspace tests
         run: |
           pytest tests/workspace ${{ matrix.pytest_args }}
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       - name: Install pipeline and sources dependencies
-        run: uv sync ${{ matrix.uv_sync_args }} --extra duckdb --extra cli --extra parquet --extra deltalake --extra sql_database --group sentry-sdk --group pipeline --group sources --group ibis
-        if: matrix.python-version != '3.14.0-beta.4'
+        run: uv sync ${{ matrix.uv_sync_args }} --extra http --extra duckdb --extra cli --extra parquet --extra deltalake --extra sql_database --group sentry-sdk --group pipeline --group sources --group ibis
+        if: matrix.python-version != '3.14'
 
       - name: Run extract and pipeline tests
         run: |
           pytest tests/extract tests/pipeline tests/libs tests/destinations tests/sources ${{ matrix.pytest_args }}
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       # here we upgrade sql alchemy to 2 an run the sql_database tests again
       - name: Upgrade sql alchemy
         run: uv run pip install sqlalchemy==2.0.32
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
 
       - name: Run extract and pipeline tests 
         run: |
           pytest tests/sources/sql_database
-        if: matrix.python-version != '3.14.0-beta.4'
+        if: matrix.python-version != '3.14'
   matrix_job_required_check:
     name: common | common tests
     needs: run_common

.github/workflows/test_tools_dashboard.yml

@@ -104,16 +104,17 @@ jobs:
         run: |
           pytest tests/workspace/helpers/dashboard
 
-      # Run pipeline dashboard e2e tests (does not pass with python 3.9, does not pass on windows (playwright does not work somehow), does not pass on python 3.13 (ibis not available))
-      # Mac is also disabled for the time being
+      # Run pipeline dashboard e2e tests (does not pass with python 3.9
       - name: Run dashboard e2e
         run: |
           marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
 
+      # note that this test will pass only when running from cmd shell (_storage\.dlt\pipelines\ must stay)
       - name: Run dashboard e2e windows
         run: |
           start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
+          timeout /t 2 /nobreak >NUL
           pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
 

Makefile

@@ -125,11 +125,6 @@ test-load-local-postgres:
 test-common:
 	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/workspace tests/load/test_dummy_client.py tests/libs tests/destinations
 
-reset-test-storage:
-	-rm -r _storage
-	mkdir _storage
-	python3 tests/tools/create_storages.py
-
 build-library: dev
 	uv version
 	uv build

README.md

@@ -31,7 +31,7 @@ Be it a Google Colab notebook, AWS Lambda function, an Airflow DAG, your local l
 
 ## Installation
 
-dlt supports Python 3.9 through Python 3.14 (beta 4). Note that some optional extras are not yet available for Python 3.14, so support for this version is considered experimental.
+dlt supports Python 3.9 through Python 3.14. Note that some optional extras are not yet available for Python 3.14, so support for this version is considered experimental.
 
 ```sh
 pip install dlt

dlt/_workspace/cli/_deploy_command.py

@@ -80,6 +80,7 @@ def __init__(
         run_on_push: bool = False,
         run_manually: bool = False,
         branch: Optional[str] = None,
+        **kwargs: Any,
     ):
         super().__init__(pipeline_script_path, location, branch)
         self.schedule = schedule
@@ -264,6 +265,7 @@ def __init__(
         location: str,
         branch: Optional[str] = None,
         secrets_format: Optional[str] = None,
+        **kwargs: Any,
     ):
         super().__init__(pipeline_script_path, location, branch)
         self.secrets_format = secrets_format

dlt/_workspace/cli/_dlt.py

@@ -1,6 +1,6 @@
+import sys
 from typing import Any, Sequence, Type, cast, List, Dict, Tuple
 import argparse
-import click
 import rich_argparse
 from rich.markdown import Markdown
 
@@ -134,6 +134,15 @@ def _create_parser() -> Tuple[argparse.ArgumentParser, Dict[str, SupportsCliComm
             " clear enough."
         ),
     )
+    parser.add_argument(
+        "--no-pwd",
+        default=False,
+        action="store_true",
+        help=(
+            "Do not add current working directory to sys.path. By default $pwd is added to "
+            "reproduce Python behavior when running scripts."
+        ),
+    )
     subparsers = parser.add_subparsers(title="Available subcommands", dest="command")
 
     # load plugins
@@ -190,6 +199,9 @@ def main() -> int:
             # switch to non-interactive if tty not connected
             with maybe_no_stdin():
                 display_run_context_info()
+                if not args.no_pwd:
+                    if "" not in sys.path:
+                        sys.path.insert(0, "")
                 cmd.execute(args)
         except Exception as ex:
             docs_url = cmd.docs_url if hasattr(cmd, "docs_url") else DEFAULT_DOCS_URL
@@ -204,7 +216,7 @@ def main() -> int:
 
             # print exception if available
             if raiseable_exception:
-                click.secho(str(ex), err=True, fg="red")
+                fmt.secho(str(ex), err=True, fg="red")
 
             fmt.note("Please refer to our docs at '%s' for further assistance." % docs_url)
             if _debug.is_debug_enabled() and raiseable_exception:

dlt/_workspace/cli/utils.py

@@ -37,11 +37,8 @@ def display_run_context_info() -> None:
         if run_context.default_profile != run_context.profile:
             # print warning
             fmt.echo(
-                "Profile %s activated on %s"
-                % (
-                    fmt.style(run_context.profile, fg="yellow", reset=True),
-                    fmt.bold(run_context.name),
-                ),
+                "Profile `%s` is active."
+                % (fmt.style(run_context.profile, fg="yellow", reset=True),),
                 err=True,
             )
 

dlt/common/runners/pool_runner.py

@@ -201,10 +201,12 @@ def _run_func() -> bool:
     try:
         logger.debug("Running pool")
         while _run_func():
-            # for next run
+            # raise on signal: safe to do that out of _run_func()
             signals.raise_if_signalled()
             runs_count += 1
             sleep(config.run_sleep)
+            # signal could come
+            signals.raise_if_signalled()
         return runs_count
     except SignalReceivedException as sigex:
         # sleep this may raise SignalReceivedException

dlt/common/runtime/collector.py

@@ -53,6 +53,7 @@ class DictCollector(Collector):
     """A collector that just counts"""
 
     def __init__(self) -> None:
+        self.step = None
         self.counters: DefaultDict[str, int] = None
 
     def update(
@@ -101,6 +102,7 @@ def __init__(
             log_level (str, optional): Log level for the logger. Defaults to INFO level
             dump_system_stats (bool, optional): Log memory and cpu usage. Defaults to True
         """
+        self.step = None
         self.log_period = log_period
         self.logger = logger
         self.log_level = log_level
@@ -260,6 +262,7 @@ def __init__(self, single_bar: bool = False, **tqdm_kwargs: Any) -> None:
             raise MissingDependencyException(
                 "TqdmCollector", ["tqdm"], "We need tqdm to display progress bars."
             )
+        self.step = None
         self.single_bar = single_bar
         self._bars: Dict[str, tqdm[None]] = {}
         self.tqdm_kwargs = tqdm_kwargs or {}
@@ -321,6 +324,7 @@ def __init__(self, single_bar: bool = True, **alive_kwargs: Any) -> None:
                 ["alive-progress"],
                 "We need alive-progress to display progress bars.",
             )
+        self.step = None
         self.single_bar = single_bar
         self._bars: Dict[str, Any] = {}
         self._bars_counts: Dict[str, int] = {}
@@ -399,6 +403,7 @@ def __init__(self, single_bar: bool = False, **enlighten_kwargs: Any) -> None:
                 ["enlighten"],
                 "We need enlighten to display progress bars with a space for log messages.",
             )
+        self.step = None
         self.single_bar = single_bar
         self.enlighten_kwargs = enlighten_kwargs
 

dlt/common/runtime/signals.py

@@ -1,30 +1,74 @@
+import sys
 import threading
 import signal
 from contextlib import contextmanager
 from threading import Event
-from typing import Any, Iterator
+from types import FrameType
+from typing import Any, Callable, Dict, Iterator, Optional, Union
 
 from dlt.common import logger
 from dlt.common.exceptions import SignalReceivedException
 
 _received_signal: int = 0
 exit_event = Event()
+_signal_counts: Dict[int, int] = {}
+_original_handlers: Dict[int, Union[int, Callable[[int, Optional[FrameType]], Any]]] = {}
 
 
-def signal_receiver(sig: int, frame: Any) -> None:
-    global _received_signal
+def signal_receiver(sig: int, frame: FrameType) -> None:
+    """Handle POSIX signals with two-stage escalation.
 
-    logger.info(f"Signal {sig} received")
+    This handler is installed by delayed_signals(). On the first occurrence of a
+    supported signal (eg. SIGINT, SIGTERM) it requests a graceful shutdown by
+    setting a process-wide flag and waking sleeping threads via exit_event.
+    A second occurrence of the same signal escalates by delegating to the
+    original handler or the system default, which typically results in an
+    immediate process termination (eg. KeyboardInterrupt for SIGINT).
 
-    if _received_signal > 0:
-        logger.info(f"Another signal received after {_received_signal}")
-        return
+    Args:
+        sig: Signal number (for example, signal.SIGINT or signal.SIGTERM).
+        frame: The current stack frame when the signal was received.
 
-    _received_signal = sig
-    # awake all threads sleeping on event
-    exit_event.set()
+    Notes:
+        - The CPython runtime delivers signal handlers in the main thread only.
+          Worker threads must cooperatively observe shutdown via raise_if_signalled()
+          or the signal-aware sleep().
+    """
+    global _received_signal
 
-    logger.info("Sleeping threads signalled")
+    # track how many times this signal type has been received
+    _signal_counts[sig] = _signal_counts.get(sig, 0) + 1
+
+    if _signal_counts[sig] == 1:
+        # first signal of this type: set flag and wake threads
+        _received_signal = sig
+        if sig == signal.SIGINT:
+            sig_desc = "CTRL-C"
+        else:
+            sig_desc = f"Signal {sig}"
+        msg = (
+            f"{sig_desc} received. Trying to shut down gracefully. It may take time to drain job"
+            f" pools. Send {sig_desc} again to force stop."
+        )
+        if sys.stdin.isatty():
+            # log to console
+            sys.stderr.write(msg)
+            sys.stderr.flush()
+        else:
+            logger.warning(msg)
+    elif _signal_counts[sig] >= 2:
+        # Second signal of this type: call original handler
+        logger.debug(f"Second signal {sig} received, calling default handler")
+        original_handler = _original_handlers.get(sig, signal.SIG_DFL)
+        if callable(original_handler):
+            original_handler(sig, frame)
+        elif original_handler == signal.SIG_DFL:
+            # Restore default and re-raise to trigger default behavior
+            signal.signal(sig, signal.SIG_DFL)
+            signal.raise_signal(sig)
+
+    exit_event.set()
+    logger.debug("Sleeping threads signalled")
 
 
 def raise_if_signalled() -> None:
@@ -38,38 +82,61 @@ def signal_received() -> bool:
 
 
 def sleep(sleep_seconds: float) -> None:
-    """A signal-aware version of sleep function. Will raise SignalReceivedException if signal was received during sleep period."""
-    # do not allow sleeping if signal was received
-    raise_if_signalled()
+    """A signal-aware version of sleep function. Will wake up if signal is received but will not raise exception."""
     # sleep or wait for signal
     exit_event.clear()
     exit_event.wait(sleep_seconds)
-    # if signal then raise
-    raise_if_signalled()
 
 
 def wake_all() -> None:
     """Wakes all threads sleeping on event"""
     exit_event.set()
 
 
+def _clear_signals() -> None:
+    global _received_signal
+
+    _received_signal = 0
+    _signal_counts.clear()
+    _original_handlers.clear()
+
+
 @contextmanager
 def delayed_signals() -> Iterator[None]:
-    """Will delay signalling until `raise_if_signalled` is used or signalled `sleep`"""
+    """Will delay signalling until `raise_if_signalled` is explicitly used or when
+    a second signal with the same int value arrives.
+
+    A no-op when not called on main thread.
+
+    Can be nested - nested calls are no-ops.
+    """
 
     if threading.current_thread() is threading.main_thread():
-        original_sigint_handler = signal.getsignal(signal.SIGINT)
+        # check if handlers are already installed (nested call)
+        current_sigint_handler = signal.getsignal(signal.SIGINT)
+
+        if current_sigint_handler is signal_receiver:
+            # already installed, this is a nested call - just yield
+            yield
+            return
+
+        # First call - install handlers
+        original_sigint_handler = current_sigint_handler
         original_sigterm_handler = signal.getsignal(signal.SIGTERM)
+
+        # store original handlers for signal_receiver to use
+        _original_handlers[signal.SIGINT] = original_sigint_handler
+        _original_handlers[signal.SIGTERM] = original_sigterm_handler
+
         try:
             signal.signal(signal.SIGINT, signal_receiver)
             signal.signal(signal.SIGTERM, signal_receiver)
             yield
         finally:
-            global _received_signal
-
-            _received_signal = 0
             signal.signal(signal.SIGINT, original_sigint_handler)
             signal.signal(signal.SIGTERM, original_sigterm_handler)
+            _clear_signals()
+
     else:
         logger.info("Running in daemon thread, signals not enabled")
         yield