Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
**10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter.

**10/07/2026:** **Bug fix:** a `".."` endpoint in a two-value date range -- `time=["2024-01-01", ".."]`, the open-ended form shown in the `waterdata.get_ratings()` docstring -- discarded the whole range, because `..` was not recognized as an open bound and failed to parse as a date. `waterdata.get_ratings()` then searched with no `datetime` at all and returned every rating regardless of date (59 instead of 44 for the docstring's bounding box from 2026-09-20 on), and the OGC getters (`get_daily()`, `get_continuous()`, `get_field_measurements()`, and the other `time`, `begin`, `end`, and `last_modified` arguments) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format". `".."` is now an open bound like `None`, so `["2024-01-01", ".."]` sends the same range as `"2024-01-01/.."` and `["2024-01-01", None]`.

**10/03/2026:** `waterdata.get_field_measurements()` and
Expand Down
3 changes: 2 additions & 1 deletion dataretrieval/ngwmn.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,8 @@ def get_water_level(
Vertical datum of the reported water level.
datetime : str or iterable of str, optional
Temporal filter — a single instant or a two-element ``[start, end]``
range (ISO-8601 dates/datetimes); ``".."`` denotes an open end.
range (ISO-8601 dates/datetimes); ``None`` or ``".."`` denotes an
open end.
properties : str or iterable of str, optional
Subset of columns to return. ``None`` (default) returns all columns.
limit : int, optional
Expand Down
34 changes: 18 additions & 16 deletions dataretrieval/ogc/dates.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,22 @@ def _is_blank(dt: str | None) -> bool:
return dt is None or bool(pd.isna(dt)) or dt in ("", _OPEN_BOUND)


def _format_one(dt: str | None, *, date: bool) -> str | None:
"""Format a single datetime element for inclusion in the API time arg."""
def _format_one(dt: str | None, *, date: bool, name: str) -> str:
"""Format a single datetime element for inclusion in the API time arg.

Raises ``ValueError`` naming *name* when the element is not blank and
matches no supported format.
"""
if dt is None or _is_blank(dt):
return _OPEN_BOUND
parsed = _parse_datetime(dt)
if parsed is None:
return None
raise ValueError(
f"{name} could not be read as a date or datetime: {dt!r}. "
"Pass an ISO 8601 date or datetime such as '2024-01-01' or "
"'2024-01-01T12:00:00Z', and None for an open end of a range "
"(['2024-01-01', None])."
)
if date:
return parsed.strftime("%Y-%m-%d")
# Naive inputs are interpreted in the system local zone (for backwards
Expand Down Expand Up @@ -155,24 +164,24 @@ def _format_api_dates(
Returns
-------
Union[str, None]
- If input is a single value, returns the formatted date/datetime string
or None if parsing fails.
- If input is a single value, returns the formatted date/datetime string.
- If input is a list of two values, returns a date/datetime range string
separated by "/" (e.g., "YYYY-MM-DD/YYYY-MM-DD" or
"YYYY-MM-DDTHH:MM:SSZ/YYYY-MM-DDTHH:MM:SSZ").
- Returns None if input is empty, all NA, or cannot be parsed.
- Returns None if input is None, empty, or every element is blank.

Raises
------
ValueError
If `datetime_input` contains more than two values.
If `datetime_input` contains more than two values, or an element that
is not blank matches no supported format.

Notes
-----
- A single blank/NA value returns None. In a two-value range, a blank/NA
or ``".."`` endpoint is rendered as ``".."`` to denote an open bound
(e.g. ``"2024-01-01/.."``); the range is only None when *every* element
is blank/NA/``".."`` or any other element fails to parse.
is blank/NA/``".."``.
- Supports ISO 8601 durations such as "P7D" and "PT36H" and pre-formatted
intervals containing ``"/"``; both are passed through unchanged.
- Converts datetimes to UTC and formats as ISO 8601 with 'Z' suffix when
Expand All @@ -199,11 +208,4 @@ def _format_api_dates(
if len(items) == 1 and isinstance(items[0], str) and _is_passthrough(items[0]):
return items[0]

# Format each element; any element that fails to parse invalidates the range.
formatted: list[str] = []
for dt in items:
one = _format_one(dt, date=date)
if one is None:
return None
formatted.append(one)
return "/".join(formatted)
return "/".join(_format_one(dt, date=date, name=name) for dt in items)
8 changes: 8 additions & 0 deletions dataretrieval/waterdata/measurements.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ def get_field_measurements(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down Expand Up @@ -146,6 +148,8 @@ def get_field_measurements(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down Expand Up @@ -438,6 +442,8 @@ def get_channel(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or "PT36H"
for the last 36 hours

Expand Down Expand Up @@ -491,6 +497,8 @@ def get_channel(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down
6 changes: 6 additions & 0 deletions dataretrieval/waterdata/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,8 @@ def get_time_series_metadata(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or "PT36H"
for the last 36 hours

Expand All @@ -516,6 +518,8 @@ def get_time_series_metadata(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand All @@ -537,6 +541,8 @@ def get_time_series_metadata(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down
6 changes: 4 additions & 2 deletions dataretrieval/waterdata/ratings.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,10 @@ def get_ratings(
where each rating came from.
time : string or list of strings, optional
STAC ``datetime`` filter (passed through verbatim under that name)
— a single date / datetime, or an interval (``"start/end"``,
optionally half-bounded with ``..``). ISO 8601 *durations*
— a single date / datetime, an interval (``"start/end"``,
optionally half-bounded with ``..``), or a list of two values
with ``None`` for an open end (``["2026-04-29", None]``). A value
that is not a date raises ``ValueError``. ISO 8601 *durations*
(``"P1M"``, ``"PT36H"``, …) are **not** supported by the
rating-curve service; passing one raises ``ValueError``.
bbox : list of numbers, optional
Expand Down
16 changes: 16 additions & 0 deletions dataretrieval/waterdata/time_series.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@ def get_daily(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand All @@ -152,6 +154,8 @@ def get_daily(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down Expand Up @@ -374,6 +378,8 @@ def get_continuous(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand All @@ -391,6 +397,8 @@ def get_continuous(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down Expand Up @@ -572,6 +580,8 @@ def get_latest_continuous(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand All @@ -593,6 +603,8 @@ def get_latest_continuous(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down Expand Up @@ -787,6 +799,8 @@ def get_latest_daily(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand All @@ -808,6 +822,8 @@ def get_latest_daily(
* A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z"
* Half-bounded intervals: "2018-02-12T00:00:00Z/.." or
"../2018-03-18T12:31:12Z"
* A list of two values: ["2018-02-12", "2018-03-18"], with None
for an open end: ["2018-02-12", None]
* Duration objects: "P1M" for data from the past month or
"PT36H" for the last 36 hours

Expand Down
12 changes: 12 additions & 0 deletions tests/waterdata_ratings_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,18 @@ def test_get_ratings_keeps_a_dotdot_open_bound_in_time(httpx_mock):
assert params["datetime"] == ["2026-04-29T00:00:00Z/.."]


def test_get_ratings_rejects_an_unreadable_time_before_any_request(httpx_mock):
"""An unreadable bound used to drop the ``datetime`` filter, so the search
silently returned every rating regardless of date."""
with pytest.raises(ValueError, match=r"^time could not be read as a date"):
get_ratings(
monitoring_location_id="USGS-01104475",
time=["2026-04-29", "tomorrow"],
download_and_parse=False,
)
assert httpx_mock.get_requests() == []


def test_get_ratings_multi_type_filters_via_property(httpx_mock, tmp_path):
"""File_type list: server filter omits it; local filter reads the property."""
httpx_mock.add_response(
Expand Down
12 changes: 12 additions & 0 deletions tests/waterdata_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,18 @@ def test_get_daily_keeps_a_dotdot_open_bound_in_time(httpx_mock):
assert sent["time"] == ["2025-01-01/.."]


@pytest.mark.parametrize(
"time", ["2025-13-45", ["2025-01-01", "yesterday"]], ids=["single", "range"]
)
def test_get_daily_rejects_an_unreadable_time_before_any_request(httpx_mock, time):
"""A bound that matches no date format used to drop the whole filter, so
the service received an empty ``time=`` and answered HTTP 400 without
naming the argument. It now fails locally, naming ``time``."""
with pytest.raises(ValueError, match=r"^time could not be read as a date"):
get_daily(monitoring_location_id="USGS-05427718", time=time)
assert httpx_mock.get_requests() == []


def test_get_daily_value_is_float_when_every_value_is_whole(httpx_mock):
"""Issue #428: whole-number values used to infer ``int64``, so ``value``
changed dtype between calls. It is ``float64`` regardless of the data."""
Expand Down
30 changes: 26 additions & 4 deletions tests/waterdata_utils_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -930,7 +930,6 @@ def test_type_cols_warning_is_singular_for_one_value():
("2018-02-12T00:00:00Z/..", False, "2018-02-12T00:00:00Z/.."),
("P7D", False, "P7D"),
("PT36H", False, "PT36H"),
("Apr", False, None),
("2024-01-01", True, "2024-01-01"),
(["2024-01-01", "2024-02-01"], True, "2024-01-01/2024-02-01"),
("2024-01-01 00:00:00", True, "2024-01-01"),
Expand All @@ -952,7 +951,6 @@ def test_type_cols_warning_is_singular_for_one_value():
"passthrough_interval",
"passthrough_duration",
"time_only_duration",
"word_with_p_not_duration",
"date_only",
"date_only_pair",
"space_separated",
Expand All @@ -966,8 +964,8 @@ def test_type_cols_warning_is_singular_for_one_value():
def test_format_api_dates(value, date, expected):
"""``_format_api_dates`` normalizes ISO 8601 datetimes to UTC (dropping
fractional seconds, converting offsets), joins a pair into an interval,
passes durations / intervals through unchanged, renders a None endpoint as
``..``, and returns None for a non-date word (e.g. ``"Apr"``)."""
passes durations / intervals through unchanged, and renders a None
endpoint as ``..``."""
assert _format_api_dates(value, date=date) == expected


Expand All @@ -984,6 +982,30 @@ def test_format_api_dates_treats_an_all_blank_sequence_as_no_filter():
assert _format_api_dates(["..", ".."]) is None


@pytest.mark.parametrize(
"value",
["2024-13-45", "Jan 1 2024", "Apr", ["2024-01-01", "garbage"], ["garbage", None]],
ids=[
"single",
"unsupported_format",
# Starts with "p" but is not a duration, so it is not passed through.
"word_with_p_not_duration",
"range_end",
"range_start",
],
)
def test_format_api_dates_rejects_an_unreadable_bound(value):
"""An element that is not blank and matches no format used to return None,
which the callers send as no filter at all. The message names the caller's
argument and the bad value, and shows the forms that are accepted."""
with pytest.raises(ValueError) as excinfo:
_format_api_dates(value, name="last_modified")
message = str(excinfo.value)
assert message.startswith("last_modified could not be read as a date")
assert "'2024-01-01'" in message
assert "None for an open end" in message


def test_format_api_dates_rejects_more_than_two_values():
"""A date filter is an instant, a duration, or a closed interval. Three
values is a caller who meant something else, and the message says which
Expand Down
Loading