Skip to content

feat: expose str_to_datetime jinja macro #351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Feb 20, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions airbyte_cdk/sources/declarative/declarative_component_schema.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3744,6 +3744,16 @@ interpolation:
- "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}"
- "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}"
- "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ', '%a, %d %b %Y %H:%M:%S %z') }}"
- title: str_to_datetime
description: Converts a string to a datetime object with UTC timezone.
arguments:
s: The string to convert.
return_type: datetime.datetime
examples:
- "{{ str_to_datetime('2022-01-14') }}"
- "{{ str_to_datetime('2022-01-01 13:45:30') }}"
- "{{ str_to_datetime('2022-01-01T13:45:30+00:00') }}"
- "{{ str_to_datetime('2022-01-01T13:45:30.123456Z') }}"
filters:
- title: hash
description: Convert the specified value to a hashed string.
Expand Down
2 changes: 1 addition & 1 deletion airbyte_cdk/sources/declarative/interpolation/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
Expand Down
3 changes: 2 additions & 1 deletion airbyte_cdk/sources/declarative/interpolation/filters.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

import base64
import hashlib
import json
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

from dataclasses import InitVar, dataclass
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#


Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#


Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

from dataclasses import InitVar, dataclass
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#


from abc import ABC, abstractmethod
from typing import Any, Optional

Expand Down
2 changes: 1 addition & 1 deletion airbyte_cdk/sources/declarative/interpolation/jinja.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

import ast
Expand Down
23 changes: 19 additions & 4 deletions airbyte_cdk/sources/declarative/interpolation/macros.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

import builtins
Expand Down Expand Up @@ -63,10 +63,24 @@ def timestamp(dt: Union[float, str]) -> Union[int, float]:
if isinstance(dt, (int, float)):
return int(dt)
else:
return _str_to_datetime(dt).astimezone(pytz.utc).timestamp()
return str_to_datetime(dt).astimezone(pytz.utc).timestamp()


def _str_to_datetime(s: str) -> datetime.datetime:
def str_to_datetime(s: str) -> datetime.datetime:
"""
Converts a string to a datetime object with UTC timezone

If the input string does not contain timezone information, UTC is assumed.
Supports both basic date strings like "2022-01-14" and datetime strings with optional timezone
like "2022-01-01T13:45:30+00:00".

Usage:
`"{{ str_to_datetime('2022-01-14') }}"`

:param s: string to parse as datetime
:return: datetime object in UTC timezone
"""

parsed_date = parser.isoparse(s)
if not parsed_date.tzinfo:
# Assume UTC if the input does not contain a timezone
Expand Down Expand Up @@ -155,7 +169,7 @@ def format_datetime(
if isinstance(dt, datetime.datetime):
return dt.strftime(format)
dt_datetime = (
datetime.datetime.strptime(dt, input_format) if input_format else _str_to_datetime(dt)
datetime.datetime.strptime(dt, input_format) if input_format else str_to_datetime(dt)
)
if format == "%s":
return str(int(dt_datetime.timestamp()))
Expand All @@ -172,5 +186,6 @@ def format_datetime(
duration,
format_datetime,
today_with_timezone,
str_to_datetime,
]
macros = {f.__name__: f for f in _macros_list}
46 changes: 45 additions & 1 deletion unit_tests/sources/declarative/interpolation/test_macros.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

import datetime
Expand Down Expand Up @@ -120,3 +120,47 @@ def test_utc_datetime_to_local_timestamp_conversion():
This test ensures correct timezone handling independent of the timezone of the system on which the sync is running.
"""
assert macros["format_datetime"](dt="2020-10-01T00:00:00Z", format="%s") == "1601510400"


@pytest.mark.parametrize(
"test_name, input_value, expected_output",
[
(
"test_basic_date",
"2022-01-14",
datetime.datetime(2022, 1, 14, tzinfo=datetime.timezone.utc),
),
(
"test_datetime_with_time",
"2022-01-01 13:45:30",
datetime.datetime(2022, 1, 1, 13, 45, 30, tzinfo=datetime.timezone.utc),
),
(
"test_datetime_with_timezone",
"2022-01-01T13:45:30+00:00",
datetime.datetime(2022, 1, 1, 13, 45, 30, tzinfo=datetime.timezone.utc),
),
(
"test_datetime_with_timezone_offset",
"2022-01-01T13:45:30+05:30",
datetime.datetime(2022, 1, 1, 8, 15, 30, tzinfo=datetime.timezone.utc),
),
(
"test_datetime_with_microseconds",
"2022-01-01T13:45:30.123456Z",
datetime.datetime(2022, 1, 1, 13, 45, 30, 123456, tzinfo=datetime.timezone.utc),
),
],
)
def test_give_valid_date_str_to_datetime_returns_datetime_object(
test_name, input_value, expected_output
):
str_to_datetime_fn = macros["str_to_datetime"]
actual_output = str_to_datetime_fn(input_value)
assert actual_output == expected_output


def test_given_invalid_date_str_to_datetime_raises_value_error():
str_to_datetime_fn = macros["str_to_datetime"]
with pytest.raises(ValueError):
str_to_datetime_fn("invalid-date")
Loading