Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 0 additions & 19 deletions airflow/decorators/__init__.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,6 @@ class TaskDecoratorCollection:
show_return_value_in_logs: bool = True,
env_vars: dict[str, str] | None = None,
inherit_env: bool = True,
use_dill: bool = False,
use_airflow_context: bool = False,
**kwargs,
) -> TaskDecorator:
Expand All @@ -145,9 +144,6 @@ class TaskDecoratorCollection:
this requires to include cloudpickle in your requirements.
- ``"dill"``: Use dill for serialize more complex types,
this requires to include dill in your requirements.
:param use_dill: Whether to use dill to serialize
the args and result (pickle is default). This allow more complex types
but requires you to include dill in your requirements.
:param system_site_packages: Whether to include
system_site_packages in your virtual environment.
See virtualenv documentation for more information.
Expand Down Expand Up @@ -176,9 +172,6 @@ class TaskDecoratorCollection:
environment. If set to ``True``, the virtual environment will inherit the environment variables
of the parent process (``os.environ``). If set to ``False``, the virtual environment will be
executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
:param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable.
"""
@overload
Expand All @@ -195,7 +188,6 @@ class TaskDecoratorCollection:
show_return_value_in_logs: bool = True,
env_vars: dict[str, str] | None = None,
inherit_env: bool = True,
use_dill: bool = False,
use_airflow_context: bool = False,
**kwargs,
) -> TaskDecorator:
Expand Down Expand Up @@ -227,9 +219,6 @@ class TaskDecoratorCollection:
environment. If set to ``True``, the virtual environment will inherit the environment variables
of the parent process (``os.environ``). If set to ``False``, the virtual environment will be
executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
:param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable.
"""
@overload
Expand Down Expand Up @@ -263,7 +252,6 @@ class TaskDecoratorCollection:
index_urls: None | Collection[str] | str = None,
venv_cache_path: None | str = None,
show_return_value_in_logs: bool = True,
use_dill: bool = False,
use_airflow_context: bool = False,
**kwargs,
) -> TaskDecorator:
Expand Down Expand Up @@ -303,9 +291,6 @@ class TaskDecoratorCollection:
logs. Defaults to True, which allows return value log output.
It can be set to False to prevent log output of return value when you return huge data
such as transmission a large amount of XCom to TaskAPI.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
:param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable.
"""
@overload
Expand All @@ -321,7 +306,6 @@ class TaskDecoratorCollection:
serializer: Literal["pickle", "cloudpickle", "dill"] | None = None,
templates_dict: Mapping[str, Any] | None = None,
show_return_value_in_logs: bool = True,
use_dill: bool = False,
**kwargs,
) -> TaskDecorator:
"""Create a decorator to wrap the decorated callable into a BranchExternalPythonOperator.
Expand Down Expand Up @@ -349,9 +333,6 @@ class TaskDecoratorCollection:
logs. Defaults to True, which allows return value log output.
It can be set to False to prevent log output of return value when you return huge data
such as transmission a large amount of XCom to TaskAPI.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
"""
@overload
def branch_external_python(
Expand Down
7 changes: 0 additions & 7 deletions providers/src/airflow/providers/standard/operators/bash.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import os
import shutil
import tempfile
import warnings
from collections.abc import Container, Sequence
from functools import cached_property
from typing import TYPE_CHECKING, Any, Callable, cast
Expand Down Expand Up @@ -158,7 +157,6 @@ def __init__(
env: dict[str, str] | None = None,
append_env: bool = False,
output_encoding: str = "utf-8",
skip_exit_code: int | None = None,
skip_on_exit_code: int | Container[int] | None = 99,
cwd: str | None = None,
output_processor: Callable[[str], Any] = lambda result: result,
Expand All @@ -168,11 +166,6 @@ def __init__(
self.bash_command = bash_command
self.env = env
self.output_encoding = output_encoding
if skip_exit_code is not None:
warnings.warn(
"skip_exit_code is deprecated. Please use skip_on_exit_code", DeprecationWarning, stacklevel=2
)
skip_on_exit_code = skip_exit_code
self.skip_on_exit_code = (
skip_on_exit_code
if isinstance(skip_on_exit_code, Container)
Expand Down
25 changes: 0 additions & 25 deletions providers/src/airflow/providers/standard/operators/python.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import sys
import textwrap
import types
import warnings
from abc import ABCMeta, abstractmethod
from collections.abc import Collection, Container, Iterable, Mapping, Sequence
from functools import cache
Expand All @@ -41,7 +40,6 @@
AirflowException,
AirflowSkipException,
DeserializingResultError,
RemovedInAirflow3Warning,
)
from airflow.models.baseoperator import BaseOperator
from airflow.models.skipmixin import SkipMixin
Expand Down Expand Up @@ -416,7 +414,6 @@ def __init__(
skip_on_exit_code: int | Container[int] | None = None,
env_vars: dict[str, str] | None = None,
inherit_env: bool = True,
use_dill: bool = False,
use_airflow_context: bool = False,
**kwargs,
):
Expand All @@ -438,18 +435,6 @@ def __init__(
)
self.string_args = string_args or []

if use_dill:
warnings.warn(
"`use_dill` is deprecated and will be removed in a future version. "
"Please provide serializer='dill' instead.",
RemovedInAirflow3Warning,
stacklevel=3,
)
if serializer:
raise AirflowException(
"Both 'use_dill' and 'serializer' parameters are set. Please set only one of them"
)
serializer = "dill"
serializer = serializer or "pickle"
if serializer not in _SERIALIZERS:
msg = (
Expand Down Expand Up @@ -664,9 +649,6 @@ class PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
environment. If set to ``True``, the virtual environment will inherit the environment variables
of the parent process (``os.environ``). If set to ``False``, the virtual environment will be
executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
:param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable.
"""

Expand Down Expand Up @@ -695,7 +677,6 @@ def __init__(
venv_cache_path: None | os.PathLike[str] = None,
env_vars: dict[str, str] | None = None,
inherit_env: bool = True,
use_dill: bool = False,
use_airflow_context: bool = False,
**kwargs,
):
Expand Down Expand Up @@ -750,7 +731,6 @@ def __init__(
skip_on_exit_code=skip_on_exit_code,
env_vars=env_vars,
inherit_env=inherit_env,
use_dill=use_dill,
use_airflow_context=use_airflow_context,
**kwargs,
)
Expand Down Expand Up @@ -967,9 +947,6 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
environment. If set to ``True``, the virtual environment will inherit the environment variables
of the parent process (``os.environ``). If set to ``False``, the virtual environment will be
executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
:param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable.
"""

Expand All @@ -991,7 +968,6 @@ def __init__(
skip_on_exit_code: int | Container[int] | None = None,
env_vars: dict[str, str] | None = None,
inherit_env: bool = True,
use_dill: bool = False,
use_airflow_context: bool = False,
**kwargs,
):
Expand Down Expand Up @@ -1019,7 +995,6 @@ def __init__(
skip_on_exit_code=skip_on_exit_code,
env_vars=env_vars,
inherit_env=inherit_env,
use_dill=use_dill,
use_airflow_context=use_airflow_context,
**kwargs,
)
Expand Down
18 changes: 0 additions & 18 deletions providers/tests/standard/operators/test_python.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
from airflow.exceptions import (
AirflowException,
DeserializingResultError,
RemovedInAirflow3Warning,
)
from airflow.models.baseoperator import BaseOperator
from airflow.models.dag import DAG
Expand Down Expand Up @@ -1168,23 +1167,6 @@ def f():

self.run_as_task(f, serializer="dill", system_site_packages=False)

@DILL_MARKER
def test_add_dill_use_dill(self):
def f():
"""Ensure dill is correctly installed."""
import dill # noqa: F401

with pytest.warns(RemovedInAirflow3Warning, match="`use_dill` is deprecated and will be removed"):
self.run_as_task(f, use_dill=True, system_site_packages=False)

def test_ambiguous_serializer(self):
def f():
pass

with pytest.warns(RemovedInAirflow3Warning, match="`use_dill` is deprecated and will be removed"):
with pytest.raises(AirflowException, match="Both 'use_dill' and 'serializer' parameters are set"):
self.run_as_task(f, use_dill=True, serializer="dill")

def test_invalid_serializer(self):
def f():
"""Ensure dill is correctly installed."""
Expand Down
2 changes: 1 addition & 1 deletion providers/tests/standard/utils/test_python_virtualenv.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ def test_should_create_virtualenv_with_extra_packages_uv(self, mock_execute_in_s
)

def test_remove_task_decorator(self):
py_source = "@task.virtualenv(use_dill=True)\ndef f():\nimport funcsigs"
py_source = '@task.virtualenv(serializer="dill")\ndef f():\nimport funcsigs'
res = remove_task_decorator(python_source=py_source, task_decorator_name="@task.virtualenv")
assert res == "def f():\nimport funcsigs"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def tutorial_taskflow_api_docker_virtualenv():

# [START extract_virtualenv]
@task.virtualenv(
use_dill=True,
serializer="dill",
system_site_packages=False,
requirements=["funcsigs"],
)
Expand Down
15 changes: 0 additions & 15 deletions tests/decorators/test_python_virtualenv.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import pytest

from airflow.decorators import setup, task, teardown
from airflow.exceptions import RemovedInAirflow3Warning
from airflow.utils import timezone
from airflow.utils.state import TaskInstanceState

Expand Down Expand Up @@ -70,20 +69,6 @@ def f():

ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)

@DILL_MARKER
def test_add_dill_use_dill(self, dag_maker):
@task.virtualenv(use_dill=True, system_site_packages=False)
def f():
"""Ensure dill is correctly installed."""
import dill # noqa: F401

with pytest.warns(RemovedInAirflow3Warning, match="`use_dill` is deprecated and will be removed"):
with dag_maker(serialized=True):
ret = f()
dag_maker.create_dagrun()

ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)

def test_no_requirements(self, dag_maker):
"""Tests that the python callable is invoked on task run."""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

class TestExternalPythonDecorator:
def test_remove_task_decorator(self):
py_source = "@task.external_python(use_dill=True)\ndef f():\nimport funcsigs"
py_source = '@task.external_python(serializer="dill")\ndef f():\nimport funcsigs'
res = remove_task_decorator(python_source=py_source, task_decorator_name="@task.external_python")
assert res == "def f():\nimport funcsigs"

Expand Down