Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
**10/09/2026:** Date arguments of the OGC getters (`get_daily()`, `get_continuous()`, `waterdata.get_ratings()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument) accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. They are read like the equivalent string: an aware value is converted to UTC, a naive one is read in the local time zone as a naive string is, a `date` is midnight of that day, and `NaT` is an open bound like `None`. **Bug fix:** these values used to fail inside the package with `AttributeError: ... has no attribute 'endswith'`, or `TypeError: ... is not iterable` for a lone `datetime`, which named no argument. Any other type, such as a number, now raises `ValueError` naming the argument and the types it accepts.

**10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter.

**10/07/2026:** **Bug fix:** a `".."` endpoint in a two-value date range -- `time=["2024-01-01", ".."]`, the open-ended form shown in the `waterdata.get_ratings()` docstring -- discarded the whole range, because `..` was not recognized as an open bound and failed to parse as a date. `waterdata.get_ratings()` then searched with no `datetime` at all and returned every rating regardless of date (59 instead of 44 for the docstring's bounding box from 2026-09-20 on), and the OGC getters (`get_daily()`, `get_continuous()`, `get_field_measurements()`, and the other `time`, `begin`, `end`, and `last_modified` arguments) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format". `".."` is now an open bound like `None`, so `["2024-01-01", ".."]` sends the same range as `"2024-01-01/.."` and `["2024-01-01", None]`.
Expand Down
70 changes: 51 additions & 19 deletions dataretrieval/ogc/dates.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
from __future__ import annotations

import re
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from datetime import date as _date
from datetime import datetime
from zoneinfo import ZoneInfo

Expand Down Expand Up @@ -65,7 +66,13 @@ def _parse_datetime(value: str) -> datetime | None:
_OPEN_BOUND = ".."


def _is_blank(dt: str | None) -> bool:
# One element of a date argument: an ISO 8601 string, or a ``date``,
# ``datetime`` or ``pandas.Timestamp`` (both subclasses of ``datetime.date``),
# with ``None`` (or ``NaN`` / ``NaT``) for an open bound.
_DateLike = str | _date | None


def _is_blank(dt: _DateLike) -> bool:
"""True for a None, NaN, empty-string, or ``..`` element.

Each is a spelling of an open bound, so ``["2024-01-01", ".."]`` means the
Expand All @@ -74,22 +81,42 @@ def _is_blank(dt: str | None) -> bool:
return dt is None or bool(pd.isna(dt)) or dt in ("", _OPEN_BOUND)


def _format_one(dt: str | None, *, date: bool, name: str) -> str:
"""Format a single datetime element for inclusion in the API time arg.
def _to_datetime(dt: str | _date, *, name: str) -> datetime:
"""Read one non-blank element as a ``datetime`` (naive iff it has no zone).

Raises ``ValueError`` naming *name* when the element is not blank and
matches no supported format.
A ``date`` is midnight of that day, like the string ``"2024-01-01"``.
Raises ``ValueError`` naming *name* for an unreadable string or a value of
another type.
"""
if dt is None or _is_blank(dt):
return _OPEN_BOUND
parsed = _parse_datetime(dt)
if isinstance(dt, pd.Timestamp):
# A plain ``datetime``: ``Timestamp.astimezone()`` needs an explicit zone.
converted: datetime = dt.to_pydatetime(warn=False)
return converted
if isinstance(dt, datetime):
return dt
if isinstance(dt, _date):
return datetime(dt.year, dt.month, dt.day) # noqa: DTZ001
parsed = _parse_datetime(dt) if isinstance(dt, str) else None
if parsed is None:
raise ValueError(
f"{name} could not be read as a date or datetime: {dt!r}. "
"Pass an ISO 8601 date or datetime such as '2024-01-01' or "
"'2024-01-01T12:00:00Z', and None for an open end of a range "
"'2024-01-01T12:00:00Z' (a datetime.date, datetime.datetime or "
"pandas.Timestamp also works), and None for an open end of a range "
"(['2024-01-01', None])."
)
return parsed


def _format_one(dt: _DateLike, *, date: bool, name: str) -> str:
"""Format a single datetime element for inclusion in the API time arg.

Raises ``ValueError`` naming *name* when the element is not blank and
cannot be read as a date or datetime.
"""
if dt is None or _is_blank(dt):
return _OPEN_BOUND
parsed = _to_datetime(dt, name=name)
if date:
return parsed.strftime("%Y-%m-%d")
# Naive inputs are interpreted in the system local zone (for backwards
Expand All @@ -101,11 +128,14 @@ def _format_one(dt: str | None, *, date: bool, name: str) -> str:


def _coerce_to_list(
datetime_input: str | Sequence[str | None],
datetime_input: str | _date | Sequence[_DateLike],
name: str = "date input",
) -> list[str | None]:
) -> list[_DateLike]:
"""Normalize datetime input to a list, raising on invalid shapes."""
if isinstance(datetime_input, str):
if isinstance(datetime_input, (str, _date)) or not isinstance(
datetime_input, Iterable
):
# A lone value; anything that is not a date is rejected per element.
return [datetime_input]
if isinstance(datetime_input, Mapping):
raise TypeError(
Expand All @@ -120,13 +150,13 @@ def _is_passthrough(single: str) -> bool:
return bool(_DURATION_RE.match(single) or "/" in single)


def _all_blank(items: list[str | None]) -> bool:
def _all_blank(items: list[_DateLike]) -> bool:
"""True when every element is None, NaN, the empty string, or ``..``."""
return all(_is_blank(dt) for dt in items)


def _format_api_dates(
datetime_input: str | Sequence[str | None] | None,
datetime_input: str | _date | Sequence[_DateLike] | None,
date: bool = False,
*,
name: str = "date input",
Expand All @@ -140,9 +170,11 @@ def _format_api_dates(

Parameters
----------
datetime_input : Union[str, List[Optional[str]], None]
A single date/datetime string or a list of one or two date/datetime
strings. Accepts formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or
datetime_input : Union[str, date, List[Optional[Union[str, date]]], None]
A single date/datetime or a list of one or two of them. Each may be a
``datetime.date``, ``datetime.datetime`` or ``pandas.Timestamp`` (a
naive one is read in the local time zone, as a naive string is), or a
string. Strings accept formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or
without ``Z``/numeric offset), or relative periods (e.g., "P7D" /
"PT36H"). Range endpoints may be ``None``/``NaN``/empty or ``".."``
to denote a half-bounded range.
Expand Down Expand Up @@ -174,7 +206,7 @@ def _format_api_dates(
------
ValueError
If `datetime_input` contains more than two values, or an element that
is not blank matches no supported format.
is not blank matches no supported format or is of another type.

Notes
-----
Expand Down
12 changes: 12 additions & 0 deletions tests/waterdata_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,18 @@ def test_construct_api_requests_two_element_date_list_becomes_interval():
assert "time=2024-01-01%2F2024-01-31" in str(req.url)


def test_construct_api_requests_accepts_timestamp_bounds():
"""A ``pandas.Timestamp`` bound, as in ``time=[df.index.min(), None]``,
builds the same request as its string spelling instead of raising
``AttributeError``."""
req = _construct_api_requests(
"daily",
monitoring_location_id="USGS-05427718",
time=[pd.Timestamp("2024-01-01", tz="UTC"), None],
)
assert "time=2024-01-01%2F.." in str(req.url)


# --- mocked getter smoke tests ------------------------------------------------
# These replace what used to be ~34 live calls to the Water Data API. Each one
# serves a committed fixture (``tests/data/waterdata_ogc_fixtures.json``, two
Expand Down
80 changes: 80 additions & 0 deletions tests/waterdata_utils_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1049,6 +1049,86 @@ def test_format_api_dates_rejects_mapping():
_format_api_dates({"2024-01-01": "ignored"})


@pytest.mark.parametrize(
"value, date, expected",
[
(datetime.date(2024, 1, 1), True, "2024-01-01"),
([datetime.date(2024, 1, 1), None], True, "2024-01-01/.."),
(
[datetime.date(2024, 1, 1), datetime.date(2024, 2, 1)],
True,
"2024-01-01/2024-02-01",
),
(pd.Timestamp("2024-01-01T10:30:00", tz="UTC"), True, "2024-01-01"),
(
pd.Timestamp("2024-01-01T10:30:00", tz="UTC"),
False,
"2024-01-01T10:30:00Z",
),
(
datetime.datetime(
2024, 1, 1, 6, 0, tzinfo=datetime.timezone(datetime.timedelta(hours=-4))
),
False,
"2024-01-01T10:00:00Z",
),
(
[pd.Timestamp("2024-01-01", tz="UTC"), pd.NaT],
False,
"2024-01-01T00:00:00Z/..",
),
(
["2024-01-01T00:00:00Z", pd.Timestamp("2024-02-01", tz="UTC")],
False,
"2024-01-01T00:00:00Z/2024-02-01T00:00:00Z",
),
],
ids=[
"date",
"date_open_end",
"date_pair",
"aware_timestamp_date_only",
"aware_timestamp_to_utc",
"aware_datetime_offset_to_utc",
"nat_is_an_open_bound",
"mixed_string_and_timestamp",
],
)
def test_format_api_dates_accepts_date_and_datetime_objects(value, date, expected):
"""``datetime.date``, ``datetime.datetime`` and ``pandas.Timestamp`` used to
raise ``AttributeError`` (no ``endswith``) or ``TypeError`` (a lone datetime
is not iterable) from inside the formatter. They are read like the
equivalent string, and ``NaT`` is an open bound like ``None``."""
assert _format_api_dates(value, date=date) == expected


@pytest.mark.parametrize("date", [True, False], ids=["date_only", "datetime"])
def test_format_api_dates_reads_naive_objects_like_naive_strings(date):
"""A naive ``datetime``/``Timestamp`` is local time, the same rule a naive
string follows, so both spellings of one instant build the same filter in
any time zone. A ``date`` is midnight of that day, like ``"2024-01-01"``."""
as_string = _format_api_dates(["2024-01-01T10:00:00", None], date=date)
naive_datetime = [datetime.datetime(2024, 1, 1, 10), None] # noqa: DTZ001
assert _format_api_dates(naive_datetime, date=date) == as_string
naive_timestamp = [pd.Timestamp("2024-01-01T10:00:00"), None]
assert _format_api_dates(naive_timestamp, date=date) == as_string
assert _format_api_dates(datetime.date(2024, 1, 1), date=date) == (
_format_api_dates("2024-01-01", date=date)
)


@pytest.mark.parametrize("value", [20240101, [20240101, None], 1.5])
def test_format_api_dates_rejects_other_types_naming_the_argument(value):
"""A number is not a date. It used to fail with an ``AttributeError`` from
inside the formatter; the message now names the caller's argument and the
types it accepts."""
with pytest.raises(ValueError) as excinfo:
_format_api_dates(value, name="time")
message = str(excinfo.value)
assert message.startswith("time could not be read as a date")
assert "pandas.Timestamp" in message


def _make_response(status, body, reason=None, content_type="text/html"):
headers = {"Content-Type": content_type}
extensions = {}
Expand Down