From c41192fa1fc5c2b3e7b8414c59f656ab67bbef28 Mon Sep 17 00:00:00 2001 From: crhyatt Date: Wed, 10 Jun 2020 10:12:27 -0500 Subject: [PATCH] Upgrade pendulum to latest major version ~2.0 (#9184) --- airflow/models/dag.py | 8 ++++---- airflow/models/taskinstance.py | 6 +++--- airflow/models/xcom.py | 4 ++-- airflow/operators/latest_only_operator.py | 2 +- .../google/cloud/operators/dataproc.py | 2 +- airflow/serialization/serialized_objects.py | 9 ++++++--- airflow/settings.py | 8 ++++---- airflow/ti_deps/dep_context.py | 2 +- airflow/utils/sqlalchemy.py | 2 +- airflow/utils/timezone.py | 4 ++-- airflow/www/decorators.py | 2 +- requirements/requirements-python3.6.txt | 18 +++++++++--------- requirements/requirements-python3.7.txt | 16 ++++++++-------- requirements/requirements-python3.8.txt | 16 ++++++++-------- requirements/setup-3.6.md5 | 2 +- requirements/setup-3.7.md5 | 2 +- requirements/setup-3.8.md5 | 2 +- setup.py | 2 +- tests/jobs/test_scheduler_job.py | 13 ++++++++----- tests/models/test_dag.py | 3 +-- tests/models/test_taskinstance.py | 16 ++++++++-------- .../google/cloud/sensors/test_gcs.py | 4 ++-- tests/serialization/test_dag_serialization.py | 19 ++++++++++++------- tests/utils/test_timezone.py | 6 +++--- 24 files changed, 89 insertions(+), 79 deletions(-) diff --git a/airflow/models/dag.py b/airflow/models/dag.py index f3f6091b51b8b..af8b8af3bc708 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -412,7 +412,7 @@ def following_schedule(self, dttm): if not self.is_fixed_time_schedule(): # relative offset (eg. every 5 minutes) delta = cron.get_next(datetime) - naive - following = dttm.in_timezone(self.timezone).add_timedelta(delta) + following = dttm.in_timezone(self.timezone) + delta else: # absolute (e.g. 3 AM) naive = cron.get_next(datetime) @@ -420,7 +420,7 @@ def following_schedule(self, dttm): following = timezone.make_aware(naive, tz) return timezone.convert_to_utc(following) elif self.normalized_schedule_interval is not None: - return dttm + self.normalized_schedule_interval + return timezone.convert_to_utc(dttm + self.normalized_schedule_interval) def previous_schedule(self, dttm): """ @@ -440,7 +440,7 @@ def previous_schedule(self, dttm): if not self.is_fixed_time_schedule(): # relative offset (eg. every 5 minutes) delta = naive - cron.get_prev(datetime) - previous = dttm.in_timezone(self.timezone).subtract_timedelta(delta) + previous = dttm.in_timezone(self.timezone) - delta else: # absolute (e.g. 3 AM) naive = cron.get_prev(datetime) @@ -448,7 +448,7 @@ def previous_schedule(self, dttm): previous = timezone.make_aware(naive, tz) return timezone.convert_to_utc(previous) elif self.normalized_schedule_interval is not None: - return dttm - self.normalized_schedule_interval + return timezone.convert_to_utc(dttm - self.normalized_schedule_interval) def get_run_dates(self, start_date, end_date=None): """ diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 1defd43add5e0..514e54dbbd882 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -640,7 +640,7 @@ def get_previous_execution_date( self, state: Optional[str] = None, session: Session = None, - ) -> Optional[pendulum.datetime]: + ) -> Optional[pendulum.DateTime]: """ The execution date from property previous_ti_success. @@ -655,7 +655,7 @@ def get_previous_start_date( self, state: Optional[str] = None, session: Session = None - ) -> Optional[pendulum.datetime]: + ) -> Optional[pendulum.DateTime]: """ The start date from property previous_ti_success. @@ -666,7 +666,7 @@ def get_previous_start_date( return prev_ti and prev_ti.start_date @property - def previous_start_date_success(self) -> Optional[pendulum.datetime]: + def previous_start_date_success(self) -> Optional[pendulum.DateTime]: """ This attribute is deprecated. Please use `airflow.models.taskinstance.TaskInstance.get_previous_start_date` method. diff --git a/airflow/models/xcom.py b/airflow/models/xcom.py index 831b9c15db480..1919f536d01cf 100644 --- a/airflow/models/xcom.py +++ b/airflow/models/xcom.py @@ -21,7 +21,7 @@ import pickle from typing import Any, Iterable, Optional, Union -from pendulum import pendulum +import pendulum from sqlalchemy import Column, LargeBinary, String, and_ from sqlalchemy.orm import Query, Session, reconstructor @@ -120,7 +120,7 @@ def set( @classmethod @provide_session def get_many(cls, - execution_date: pendulum.datetime, + execution_date: pendulum.DateTime, key: Optional[str] = None, task_ids: Optional[Union[str, Iterable[str]]] = None, dag_ids: Optional[Union[str, Iterable[str]]] = None, diff --git a/airflow/operators/latest_only_operator.py b/airflow/operators/latest_only_operator.py index 0c10ffc593dd1..d19e1b047efaa 100644 --- a/airflow/operators/latest_only_operator.py +++ b/airflow/operators/latest_only_operator.py @@ -48,7 +48,7 @@ def choose_branch(self, context: Dict) -> Union[str, Iterable[str]]: "Externally triggered DAG_Run: allowing execution to proceed.") return list(context['task'].get_direct_relative_ids(upstream=False)) - now = pendulum.utcnow() + now = pendulum.now('UTC') left_window = context['dag'].following_schedule( context['execution_date']) right_window = context['dag'].following_schedule(left_window) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 8c4a0025a2bc7..9d537bf02dc08 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -295,7 +295,7 @@ def _build_lifecycle_config(self, cluster_data): if self.auto_delete_time: utc_auto_delete_time = timezone.convert_to_utc(self.auto_delete_time) cluster_data['config']['lifecycle_config']['auto_delete_time'] = \ - utc_auto_delete_time.format('%Y-%m-%dT%H:%M:%S.%fZ', formatter='classic') + utc_auto_delete_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ') elif self.auto_delete_ttl: cluster_data['config']['lifecycle_config']['auto_delete_ttl'] = \ "{}s".format(self.auto_delete_ttl) diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index adff838221dce..b04cb7757e388 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -25,6 +25,7 @@ import cattr import pendulum from dateutil import relativedelta +from pendulum.tz.timezone import Timezone from airflow.exceptions import AirflowException from airflow.models import Connection @@ -186,7 +187,7 @@ def _serialize(cls, var: Any) -> Any: # Unfortunately there is no support for r return cls._encode(var.timestamp(), type_=DAT.DATETIME) elif isinstance(var, datetime.timedelta): return cls._encode(var.total_seconds(), type_=DAT.TIMEDELTA) - elif isinstance(var, (pendulum.tz.Timezone, pendulum.tz.timezone_info.TimezoneInfo)): + elif isinstance(var, (Timezone)): return cls._encode(str(var.name), type_=DAT.TIMEZONE) elif isinstance(var, relativedelta.relativedelta): encoded = {k: v for k, v in var.__dict__.items() if not k.startswith("_") and v} @@ -212,6 +213,7 @@ def _serialize(cls, var: Any) -> Any: # Unfortunately there is no support for r except Exception: # pylint: disable=broad-except log.error('Failed to stringify.', exc_info=True) return FAILED + # pylint: enable=too-many-return-statements @classmethod @@ -239,7 +241,7 @@ def _deserialize(cls, encoded_var: Any) -> Any: # pylint: disable=too-many-retu elif type_ == DAT.TIMEDELTA: return datetime.timedelta(seconds=var) elif type_ == DAT.TIMEZONE: - return pendulum.timezone(var) + return Timezone(var) elif type_ == DAT.RELATIVEDELTA: if 'weekday' in var: var['weekday'] = relativedelta.weekday(*var['weekday']) # type: ignore @@ -252,7 +254,7 @@ def _deserialize(cls, encoded_var: Any) -> Any: # pylint: disable=too-many-retu raise TypeError('Invalid type {!s} in deserialization.'.format(type_)) _deserialize_datetime = pendulum.from_timestamp - _deserialize_timezone = pendulum.timezone + _deserialize_timezone = pendulum.tz.timezone @classmethod def _deserialize_timedelta(cls, seconds: int) -> datetime.timedelta: @@ -538,6 +540,7 @@ def __get_constructor_defaults(): # pylint: disable=no-method-argument param_to_attr.get(k, k): v.default for k, v in signature(DAG).parameters.items() if v.default is not v.empty } + _CONSTRUCTOR_PARAMS = __get_constructor_defaults.__func__() # type: ignore del __get_constructor_defaults diff --git a/airflow/settings.py b/airflow/settings.py index 25b9cfda2d01b..281469d606dc9 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -39,13 +39,13 @@ log = logging.getLogger(__name__) -TIMEZONE = pendulum.timezone('UTC') +TIMEZONE = pendulum.tz.timezone('UTC') try: tz = conf.get("core", "default_timezone") if tz == "system": - TIMEZONE = pendulum.local_timezone() + TIMEZONE = pendulum.tz.local_timezone() else: - TIMEZONE = pendulum.timezone(tz) + TIMEZONE = pendulum.tz.timezone(tz) except Exception: pass log.info("Configured default timezone %s" % TIMEZONE) @@ -214,7 +214,7 @@ def dispose_orm(): def configure_adapters(): - from pendulum import Pendulum + from pendulum import DateTime as Pendulum try: from sqlite3 import register_adapter register_adapter(Pendulum, lambda val: val.isoformat(' ')) diff --git a/airflow/ti_deps/dep_context.py b/airflow/ti_deps/dep_context.py index ae5b1db0f9a38..a4e8a7d45d2eb 100644 --- a/airflow/ti_deps/dep_context.py +++ b/airflow/ti_deps/dep_context.py @@ -83,7 +83,7 @@ def __init__( self.ignore_ti_state = ignore_ti_state self.finished_tasks = finished_tasks - def ensure_finished_tasks(self, dag, execution_date: pendulum.datetime, session: Session): + def ensure_finished_tasks(self, dag, execution_date: pendulum.DateTime, session: Session): """ This method makes sure finished_tasks is populated if it's currently None. This is for the strange feature of running tasks without dag_run. diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 73ab5484b211b..4186efae68988 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -32,7 +32,7 @@ log = logging.getLogger(__name__) -utc = pendulum.timezone('UTC') +utc = pendulum.tz.timezone('UTC') using_mysql = conf.get('core', 'sql_alchemy_conn').lower().startswith('mysql') diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index 02c9bb28eb9fd..399a5a2e2d5ed 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -23,7 +23,7 @@ from airflow.settings import TIMEZONE # UTC time zone as a tzinfo instance. -utc = pendulum.timezone('UTC') +utc = pendulum.tz.timezone('UTC') def is_localized(value): @@ -176,4 +176,4 @@ def parse(string, timezone=None): :param string: time string """ - return pendulum.parse(string, tz=timezone or TIMEZONE) + return pendulum.parse(string, tz=timezone or TIMEZONE, strict=False) diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py index b1369021443b4..68bf466c0278e 100644 --- a/airflow/www/decorators.py +++ b/airflow/www/decorators.py @@ -50,7 +50,7 @@ def wrapper(*args, **kwargs): if 'execution_date' in request.values: log.execution_date = pendulum.parse( - request.values.get('execution_date')) + request.values.get('execution_date'), strict=False) session.add(log) diff --git a/requirements/requirements-python3.6.txt b/requirements/requirements-python3.6.txt index 3ebbbe9a44d26..26f14e14d4f45 100644 --- a/requirements/requirements-python3.6.txt +++ b/requirements/requirements-python3.6.txt @@ -45,7 +45,7 @@ apispec==1.3.3 appdirs==1.4.4 argcomplete==1.11.1 asn1crypto==1.3.0 -astroid==2.3.3 +astroid==2.4.2 async-generator==1.10 async-timeout==3.0.1 atlasclient==1.0.0 @@ -66,15 +66,15 @@ azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage==0.36.0 -backcall==0.1.0 +backcall==0.2.0 bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.13.25 +boto3==1.13.26 boto==2.49.0 -botocore==1.16.25 +botocore==1.16.26 bowler==0.8.0 cached-property==1.5.1 cachetools==4.1.0 @@ -145,7 +145,7 @@ google-auth-oauthlib==0.4.1 google-auth==1.16.1 google-cloud-automl==0.10.0 google-cloud-bigquery-datatransfer==1.0.0 -google-cloud-bigquery==1.24.0 +google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==0.5.0 google-cloud-core==1.3.0 @@ -187,7 +187,7 @@ ijson==2.6.1 imagesize==1.2.0 immutables==0.14 importlib-metadata==1.6.1 -importlib-resources==2.0.0 +importlib-resources==1.5.0 inflection==0.5.0 ipdb==0.13.2 ipython-genutils==0.2.0 @@ -223,7 +223,7 @@ monotonic==1.5 more-itertools==8.3.0 moto==1.3.14 msgpack==1.0.0 -msrest==0.6.15 +msrest==0.6.16 msrestazure==0.6.3 multi-key-dict==2.0.3 multidict==4.7.6 @@ -233,7 +233,7 @@ mysql-connector-python==8.0.18 mysqlclient==1.3.14 natsort==7.0.1 nbclient==0.3.1 -nbformat==5.0.6 +nbformat==5.0.7 nest-asyncio==1.3.3 networkx==2.4 nodeenv==1.4.0 @@ -252,7 +252,7 @@ paramiko==2.7.1 parso==0.7.0 pathspec==0.8.0 pbr==5.4.5 -pendulum==1.4.4 +pendulum==2.1.0 pep562==1.0 pexpect==4.8.0 pickleshare==0.7.5 diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt index 9f5104f273765..4bb9b8701aace 100644 --- a/requirements/requirements-python3.7.txt +++ b/requirements/requirements-python3.7.txt @@ -66,15 +66,15 @@ azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage==0.36.0 -backcall==0.1.0 +backcall==0.2.0 bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.13.25 +boto3==1.13.26 boto==2.49.0 -botocore==1.16.25 +botocore==1.16.26 bowler==0.8.0 cached-property==1.5.1 cachetools==4.1.0 @@ -144,7 +144,7 @@ google-auth-oauthlib==0.4.1 google-auth==1.16.1 google-cloud-automl==0.10.0 google-cloud-bigquery-datatransfer==1.0.0 -google-cloud-bigquery==1.24.0 +google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==0.5.0 google-cloud-core==1.3.0 @@ -219,7 +219,7 @@ monotonic==1.5 more-itertools==8.3.0 moto==1.3.14 msgpack==1.0.0 -msrest==0.6.15 +msrest==0.6.16 msrestazure==0.6.3 multi-key-dict==2.0.3 multidict==4.7.6 @@ -229,7 +229,7 @@ mysql-connector-python==8.0.18 mysqlclient==1.3.14 natsort==7.0.1 nbclient==0.3.1 -nbformat==5.0.6 +nbformat==5.0.7 nest-asyncio==1.3.3 networkx==2.4 nodeenv==1.4.0 @@ -248,7 +248,7 @@ paramiko==2.7.1 parso==0.7.0 pathspec==0.8.0 pbr==5.4.5 -pendulum==1.4.4 +pendulum==2.1.0 pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 @@ -378,7 +378,7 @@ virtualenv==20.0.21 watchtower==0.7.3 wcwidth==0.2.4 websocket-client==0.57.0 -wrapt==1.12.1 +wrapt==1.11.2 xmltodict==0.12.0 yamllint==1.23.0 yandexcloud==0.41.0 diff --git a/requirements/requirements-python3.8.txt b/requirements/requirements-python3.8.txt index d6b3d45c4ac29..aa70683ee9501 100644 --- a/requirements/requirements-python3.8.txt +++ b/requirements/requirements-python3.8.txt @@ -45,7 +45,7 @@ apispec==1.3.3 appdirs==1.4.4 argcomplete==1.11.1 asn1crypto==1.3.0 -astroid==2.4.2 +astroid==2.3.3 async-generator==1.10 async-timeout==3.0.1 atlasclient==1.0.0 @@ -72,9 +72,9 @@ beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.13.25 +boto3==1.13.26 boto==2.49.0 -botocore==1.16.25 +botocore==1.16.26 bowler==0.8.0 cached-property==1.5.1 cachetools==4.1.0 @@ -144,7 +144,7 @@ google-auth-oauthlib==0.4.1 google-auth==1.16.1 google-cloud-automl==0.10.0 google-cloud-bigquery-datatransfer==1.0.0 -google-cloud-bigquery==1.24.0 +google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==0.5.0 google-cloud-core==1.3.0 @@ -219,7 +219,7 @@ monotonic==1.5 more-itertools==8.3.0 moto==1.3.14 msgpack==1.0.0 -msrest==0.6.15 +msrest==0.6.16 msrestazure==0.6.3 multi-key-dict==2.0.3 multidict==4.7.6 @@ -229,7 +229,7 @@ mysql-connector-python==8.0.18 mysqlclient==1.3.14 natsort==7.0.1 nbclient==0.3.1 -nbformat==5.0.6 +nbformat==5.0.7 nest-asyncio==1.3.3 networkx==2.4 nodeenv==1.4.0 @@ -248,7 +248,7 @@ paramiko==2.7.1 parso==0.7.0 pathspec==0.8.0 pbr==5.4.5 -pendulum==1.4.4 +pendulum==2.1.0 pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 @@ -377,7 +377,7 @@ virtualenv==20.0.21 watchtower==0.7.3 wcwidth==0.2.4 websocket-client==0.57.0 -wrapt==1.12.1 +wrapt==1.11.2 xmltodict==0.12.0 yamllint==1.23.0 yandexcloud==0.41.0 diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5 index a9a99c1d9bb73..2a56dd9d5157f 100644 --- a/requirements/setup-3.6.md5 +++ b/requirements/setup-3.6.md5 @@ -1 +1 @@ -4a78183a54d31b57adf0ad8fea8e5bc6 /opt/airflow/setup.py +72eff355caceb467d3c4e62b06f49beb /opt/airflow/setup.py diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5 index a9a99c1d9bb73..2a56dd9d5157f 100644 --- a/requirements/setup-3.7.md5 +++ b/requirements/setup-3.7.md5 @@ -1 +1 @@ -4a78183a54d31b57adf0ad8fea8e5bc6 /opt/airflow/setup.py +72eff355caceb467d3c4e62b06f49beb /opt/airflow/setup.py diff --git a/requirements/setup-3.8.md5 b/requirements/setup-3.8.md5 index a9a99c1d9bb73..2a56dd9d5157f 100644 --- a/requirements/setup-3.8.md5 +++ b/requirements/setup-3.8.md5 @@ -1 +1 @@ -4a78183a54d31b57adf0ad8fea8e5bc6 /opt/airflow/setup.py +72eff355caceb467d3c4e62b06f49beb /opt/airflow/setup.py diff --git a/setup.py b/setup.py index fae937603f171..425d671bde0aa 100644 --- a/setup.py +++ b/setup.py @@ -709,7 +709,7 @@ def is_package_excluded(package: str, exclusion_list: List[str]): 'lockfile>=0.12.2', 'markdown>=2.5.2, <3.0', 'pandas>=0.17.1, <2.0', - 'pendulum==1.4.4', + 'pendulum~=2.0', 'pep562~=1.0;python_version<"3.7"', 'psutil>=4.2.0, <6.0.0', 'pygments>=2.0.1, <3.0', diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index f57291fbeb234..70b7c35ee7b27 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -1244,6 +1244,7 @@ class TestDagFileProcessorQueriesCount(unittest.TestCase): These tests allow easy detection when a change is made that affects the performance of the DagFileProcessor. """ + def setUp(self) -> None: clear_db_runs() clear_db_pools() @@ -2353,6 +2354,7 @@ def evaluate_dagrun( run_kwargs=None, advance_execution_date=False, session=None): # pylint: disable=unused-argument + """ Helper for testing DagRun states with simple two-task DAGS. This is hackish: a dag run is created but its tasks are @@ -2518,7 +2520,7 @@ def test_scheduler_start_date(self): dag_id = 'test_start_date_scheduling' dag = self.dagbag.get_dag(dag_id) dag.clear() - self.assertGreater(dag.start_date, datetime.datetime.utcnow()) + self.assertGreater(dag.start_date, datetime.datetime.now(timezone.utc)) scheduler = SchedulerJob(dag_id, executor=self.null_exec, @@ -3385,6 +3387,7 @@ class TestSchedulerJobQueriesCount(unittest.TestCase): different DAG files. These tests allow easy detection when a change is made that affects the performance of the SchedulerJob. """ + def setUp(self) -> None: clear_db_runs() clear_db_pools() @@ -3397,9 +3400,9 @@ def setUp(self) -> None: # pylint: disable=bad-whitespace # expected, dag_count, task_count # One DAG with one task per DAG file - ( 13, 1, 1), # noqa + (13, 1, 1), # noqa # One DAG with five tasks per DAG file - ( 25, 1, 5), # noqa + (25, 1, 5), # noqa # 10 DAGs with 10 tasks per DAG file (108, 10, 10), # noqa ] @@ -3438,9 +3441,9 @@ def test_execute_queries_count_with_harvested_dags(self, expected_query_count, d # pylint: disable=bad-whitespace # expected, dag_count, task_count # One DAG with one task per DAG file - (2, 1, 1), # noqa + (2, 1, 1), # noqa # One DAG with five tasks per DAG file - (2, 1, 5), # noqa + (2, 1, 5), # noqa # 10 DAGs with 10 tasks per DAG file (2, 10, 10), # noqa ] diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 050d550ae2dc3..6e57fae45906c 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -31,7 +31,6 @@ import pendulum from dateutil.relativedelta import relativedelta from parameterized import parameterized -from pendulum import utcnow from airflow import models, settings from airflow.configuration import conf @@ -1210,7 +1209,7 @@ def test_schedule_dag_no_end_date_up_to_today_only(self): """ session = settings.Session() delta = datetime.timedelta(days=1) - now = utcnow() + now = pendulum.now('UTC') start_date = now.subtract(weeks=1) runs = (now - start_date).days diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index e578085354879..88120aa72be02 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -21,7 +21,7 @@ import time import unittest import urllib -from typing import List, Optional, Union +from typing import List, Optional, Union, cast from unittest.mock import call, mock_open, patch import pendulum @@ -591,7 +591,7 @@ def test_next_retry_datetime_short_intervals(self): date = ti.next_retry_datetime() # between 1 * 2^0.5 and 1 * 2^1 (15 and 30) - period = ti.end_date.add(seconds=1) - ti.end_date.add(seconds=15) + period = ti.end_date.add(seconds=15) - ti.end_date.add(seconds=1) self.assertTrue(date in period) def test_reschedule_handling(self): @@ -1265,12 +1265,12 @@ def _test_previous_dates_setup(schedule_interval: Union[str, datetime.timedelta, dag = models.DAG(dag_id=dag_id, schedule_interval=schedule_interval, catchup=catchup) task = DummyOperator(task_id='task', dag=dag, start_date=DEFAULT_DATE) - def get_test_ti(session, execution_date: pendulum.datetime, state: str) -> TI: + def get_test_ti(session, execution_date: pendulum.DateTime, state: str) -> TI: dag.create_dagrun( run_type=DagRunType.SCHEDULED, state=state, execution_date=execution_date, - start_date=pendulum.utcnow(), + start_date=pendulum.now('UTC'), session=session ) ti = TI(task=task, execution_date=execution_date) @@ -1279,7 +1279,7 @@ def get_test_ti(session, execution_date: pendulum.datetime, state: str) -> TI: with create_session() as session: # type: Session - date = pendulum.parse('2019-01-01T00:00:00+00:00') + date = cast(pendulum.DateTime, pendulum.parse('2019-01-01T00:00:00+00:00')) ret = [] @@ -1384,9 +1384,9 @@ def test_pendulum_template_dates(self): template_context = ti.get_template_context() - self.assertIsInstance(template_context["execution_date"], pendulum.datetime) - self.assertIsInstance(template_context["next_execution_date"], pendulum.datetime) - self.assertIsInstance(template_context["prev_execution_date"], pendulum.datetime) + self.assertIsInstance(template_context["execution_date"], pendulum.DateTime) + self.assertIsInstance(template_context["next_execution_date"], pendulum.DateTime) + self.assertIsInstance(template_context["prev_execution_date"], pendulum.DateTime) @parameterized.expand( [ diff --git a/tests/providers/google/cloud/sensors/test_gcs.py b/tests/providers/google/cloud/sensors/test_gcs.py index e3dfd619e2160..e353238d8ebe4 100644 --- a/tests/providers/google/cloud/sensors/test_gcs.py +++ b/tests/providers/google/cloud/sensors/test_gcs.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from unittest import TestCase, mock import pendulum @@ -85,7 +85,7 @@ def test_should_support_datetime(self): 'execution_date': datetime(2019, 2, 14, 0, 0) } result = ts_function(context) - self.assertEqual(datetime(2019, 2, 19, 0, 0), result) + self.assertEqual(datetime(2019, 2, 19, 0, 0, tzinfo=timezone.utc), result) def test_should_support_cron(self): dag = DAG( diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 1bb00d5d06494..7af8eb90dbe15 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -21,7 +21,7 @@ import multiprocessing import os import unittest -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from glob import glob from unittest import mock @@ -376,14 +376,17 @@ def validate_deserialized_task(self, serialized_task, task,): assert serialized_task.subdag is None @parameterized.expand([ - (datetime(2019, 8, 1), None, datetime(2019, 8, 1)), - (datetime(2019, 8, 1), datetime(2019, 8, 2), datetime(2019, 8, 2)), - (datetime(2019, 8, 1), datetime(2019, 7, 30), datetime(2019, 8, 1)), + (datetime(2019, 8, 1, tzinfo=timezone.utc), None, datetime(2019, 8, 1, tzinfo=timezone.utc)), + (datetime(2019, 8, 1, tzinfo=timezone.utc), datetime(2019, 8, 2, tzinfo=timezone.utc), + datetime(2019, 8, 2, tzinfo=timezone.utc)), + (datetime(2019, 8, 1, tzinfo=timezone.utc), datetime(2019, 7, 30, tzinfo=timezone.utc), + datetime(2019, 8, 1, tzinfo=timezone.utc)), ]) def test_deserialization_start_date(self, dag_start_date, task_start_date, expected_task_start_date): + dag = DAG(dag_id='simple_dag', start_date=dag_start_date) BaseOperator(task_id='simple_task', dag=dag, start_date=task_start_date) @@ -400,9 +403,11 @@ def test_deserialization_start_date(self, self.assertEqual(simple_task.start_date, expected_task_start_date) @parameterized.expand([ - (datetime(2019, 8, 1), None, datetime(2019, 8, 1)), - (datetime(2019, 8, 1), datetime(2019, 8, 2), datetime(2019, 8, 1)), - (datetime(2019, 8, 1), datetime(2019, 7, 30), datetime(2019, 7, 30)), + (datetime(2019, 8, 1, tzinfo=timezone.utc), None, datetime(2019, 8, 1, tzinfo=timezone.utc)), + (datetime(2019, 8, 1, tzinfo=timezone.utc), datetime(2019, 8, 2, tzinfo=timezone.utc), + datetime(2019, 8, 1, tzinfo=timezone.utc)), + (datetime(2019, 8, 1, tzinfo=timezone.utc), datetime(2019, 7, 30, tzinfo=timezone.utc), + datetime(2019, 7, 30, tzinfo=timezone.utc)), ]) def test_deserialization_end_date(self, dag_end_date, diff --git a/tests/utils/test_timezone.py b/tests/utils/test_timezone.py index 8b7031f31ceb0..400813e88a4cc 100644 --- a/tests/utils/test_timezone.py +++ b/tests/utils/test_timezone.py @@ -23,9 +23,9 @@ from airflow.utils import timezone -CET = pendulum.timezone("Europe/Paris") -EAT = pendulum.timezone('Africa/Nairobi') # Africa/Nairobi -ICT = pendulum.timezone('Asia/Bangkok') # Asia/Bangkok +CET = pendulum.tz.timezone("Europe/Paris") +EAT = pendulum.tz.timezone('Africa/Nairobi') # Africa/Nairobi +ICT = pendulum.tz.timezone('Asia/Bangkok') # Asia/Bangkok UTC = timezone.utc