Address generic radar data with specific timestamp

tests/dwd/radar/test_api_historic.py

@@ -1,10 +1,13 @@
+import re
+
 import pytest
-from datetime import datetime
+from datetime import datetime, timedelta
 from io import BytesIO
 from pathlib import Path
 
 from wetterdienst import DWDRadarRequest, TimeResolution
 from wetterdienst.dwd.radar.metadata import RadarParameter
+from wetterdienst.dwd.radar.sites import RadarSite
 
 HERE = Path(__file__).parent
 
@@ -43,3 +46,35 @@ def test_radar_request_radolan_grid_hourly():
     radolan_hourly_test = next(request.collect_data())[1]
 
     assert radolan_hourly.getvalue() == radolan_hourly_test.getvalue()
+
+
+@pytest.mark.remote
+def test_radar_request_dx_reflectivity_historic():
+    """
+    Example for testing radar DX site for a specific date.
+    """
+
+    tm = datetime.now()
+
+    # One day before.
+    tm = tm - timedelta(days=1)
+
+    # Round to the 5 minute mark before tm.
+    # https://stackoverflow.com/a/3464000
+    tm = tm - timedelta(minutes=tm.minute % 5,
+                        seconds=tm.second,
+                        microseconds=tm.microsecond)
+
+    request = DWDRadarRequest(
+        radar_parameter=RadarParameter.DX_REFLECTIVITY,
+        date_times=[tm],
+        radar_site=RadarSite.BOO,
+    )
+
+    buffer = next(request.collect_data())[1]
+    payload = buffer.getvalue()
+
+    date = bytes(tm.strftime("%d%H%M"), encoding="ascii")
+    header = b"DX" + date + b"101320920BY.....VS 2CO0CD4CS0EP0.80.80.80.80.80.80.80.8MS"
+
+    assert re.match(header, payload)

tests/dwd/radar/test_api_latest.py

@@ -20,8 +20,8 @@ def test_radar_request_rx_reflectivity_latest():
     payload = buffer.getvalue()
 
     header = (
-        b"RX......100000920BY 810134VS 3SW   2.28.1PR E\\+00INT   5GP 900x 900MS "
-        b"62<asb,boo,ros,hnr,umd,pro,ess,fld,drs,neu,oft,eis,tur,fbg,mem>"
+        b"RX......100000920BY 8101..VS 3SW   2.28.1PR E\\+00INT   5GP 900x 900MS "
+        b"..<asb,boo,ros,hnr,umd,pro,ess,fld,drs,neu,(nhb,)?oft,eis,tur,fbg,mem>"
     )
 
     assert re.match(header, payload)
@@ -42,11 +42,11 @@ def test_radar_request_rw_reflectivity_latest():
     payload = buffer.getvalue()
 
     header = (
-        b"RW......100000920BY1620145VS 3SW   2.28.1PR E-01INT  60GP 900x 900MF 00000001MS "  # noqa:E501,B950
-        b"62<asb,boo,ros,hnr,umd,pro,ess,fld,drs,neu,oft,eis,tur,fbg,mem>"
+        b"RW......100000920BY16201..VS 3SW   2.28.1PR E-01INT  60GP 900x 900MF 00000001MS "  # noqa:E501,B950
+        b"..<asb,boo,ros,hnr,umd,pro,ess,fld,drs,neu,(nhb,)?oft,eis,tur,fbg,mem>"
     )
 
-    assert re.match(header, payload)
+    assert re.match(header, payload), payload[:150]
 
 
 @pytest.mark.remote

wetterdienst/dwd/radar/access.py

@@ -107,36 +107,54 @@ def _collect_generic_radar_data(
         list of tuples of a datetime and the corresponding file in bytes
     """
 
-    # data = []
+    # Find latest file.
+    if date_times[0] == RadarDate.LATEST.value:
 
-    # FIXME: Implement indexing by timestamp when needed.
-    if date_times[0] != RadarDate.LATEST.value:
-        raise NotImplementedError(
-            "Acquisition of generic radar data only supports LATEST file for now"
+        file_index = create_fileindex_radar(
+            parameter=parameter,
+            radar_site=radar_site,
+            radar_data_type=radar_data_type,
+            parse_datetime=False,
         )
 
-    file_index = create_fileindex_radar(
-        parameter=parameter,
-        radar_site=radar_site,
-        radar_data_type=radar_data_type,
-    )
+        filenames = file_index["FILENAME"].tolist()
 
-    # Find latest file.
-    filenames = file_index["FILENAME"].tolist()
+        try:
+            latest_file = list(filter(lambda x: "-latest-" in x, filenames))[0]
+        except IndexError:
+            latest_file = filenames[-1]
 
-    try:
-        latest_file = list(filter(lambda x: "-latest-" in x, filenames))[0]
-    except IndexError:
-        latest_file = filenames[-1]
+        # Make up single-entry response.
+        # TODO: Discuss which datetime to use here.
+        # TODO: Maybe decode timestamp from file header, e.g. RX272355, RW272250, DX280005
+        now = datetime.now()
+        payload = _download_generic_data(latest_file)
+        data = [(now, payload)]
+
+    else:
+
+        file_index = create_fileindex_radar(
+            parameter=parameter,
+            radar_site=radar_site,
+            radar_data_type=radar_data_type,
+            parse_datetime=True,
+        )
+
+        data = []
+        for date_time in date_times:
 
-    # Make up single-entry response.
-    # TODO: Discuss which datetime to use here.
-    # TODO: Maybe decode timestamp from file header, e.g. RX272355, RW272250, DX280005
-    now = datetime.now()
-    payload = _download_generic_data(latest_file)
-    response = [(now, payload)]
+            # Filter by date.
+            fi = file_index[file_index[DWDMetaColumns.DATETIME.value] == date_time]
 
-    return response
+            if fi.empty:
+                log.warning(f"No radar file found for {str(date_time)}, skipping.")
+                continue
+
+            url = fi[DWDMetaColumns.FILENAME.value].item()
+            date_time_and_file = (date_time, _download_generic_data(url))
+            data.append(date_time_and_file)
+
+    return data
 
 
 @payload_cache_five_minutes.cache_on_arguments()
@@ -183,7 +201,7 @@ def _collect_radolan_grid_data(
                     (
                         date_time,
                         restore_radar_data(
-                            RadarParameter.RADOLAN, date_time, time_resolution, folder
+                            RadarParameter.RADOLAN_GRID, date_time, time_resolution, folder
                         ),
                     )
                 )
@@ -210,7 +228,7 @@ def _collect_radolan_grid_data(
 
         if write_file:
             store_radar_data(
-                RadarParameter.RADOLAN, date_time_and_file, time_resolution, folder
+                RadarParameter.RADOLAN_GRID, date_time_and_file, time_resolution, folder
             )
 
     return data

wetterdienst/dwd/radar/api.py

@@ -91,6 +91,7 @@ def __init__(
             )
 
         # Run on of two indexing variants, either for RADOLAN_GRID or generic RADOLAN.
+        # TODO: File index will be made twice, consider sharing it with ``collect_data``.
         if self.radar_parameter == RadarParameter.RADOLAN_GRID:
             file_index = create_fileindex_radolan_grid(self.time_resolution)
         else:
@@ -100,6 +101,7 @@ def __init__(
                 period_type,
                 radar_site,
                 radar_data_type,
+                parse_datetime=False,
             )
 
         self.__build_date_times(file_index, date_times, start_date, end_date)

wetterdienst/dwd/radar/index.py

@@ -21,9 +21,16 @@
 from wetterdienst.util.cache import fileindex_cache_five_minutes
 from wetterdienst.util.network import list_remote_files
 
-RADOLAN_HISTORICAL_DT_REGEX = r"(?<!\d)\d{6}(?!\d)"
+# raa01-sf_10000-2001010050-dwd---bin.gz
+# raa00-dx_10132-2009260240-boo---bin
 RADOLAN_RECENT_DT_REGEX = r"(?<!\d)\d{10}(?!\d)"
 
+# SF201901.tar.gz
+RADOLAN_HISTORICAL_DT_REGEX = r"(?<!\d)\d{6}(?!\d)"
+
+# ras07-stqual-vol5minng01_sweeph5onem_vradh_02-2020092616364300-boo-10132-hd5
+RADAR_HDF5_DT_REGEX = r"(?<!\d)\d{12}"
+
 
 @fileindex_cache_five_minutes.cache_on_arguments()
 def create_fileindex_radar(
@@ -32,6 +39,7 @@ def create_fileindex_radar(
     period_type: Optional[PeriodType] = None,
     radar_site: Optional[RadarSite] = None,
     radar_data_type: Optional[RadarDataType] = None,
+    parse_datetime: bool = False,
 ) -> pd.DataFrame:
     """
     Function to create a file index of the DWD radar data, which is shipped as
@@ -71,6 +79,30 @@ def create_fileindex_radar(
                 files_server[DWDMetaColumns.FILENAME.value].str.contains("--buf")
             ]
 
+    # Decode datetime of file for filtering.
+    if parse_datetime:
+        if radar_data_type == RadarDataType.HDF5:
+            pattern = re.compile(RADAR_HDF5_DT_REGEX)
+        else:
+            pattern = re.compile(RADOLAN_RECENT_DT_REGEX)
+
+        def get_date_from_filename(filename):
+
+            try:
+                datestr = pattern.findall(filename)[0]
+                return parse(
+                    datestr,
+                    date_formats=[DatetimeFormat.ymdhm.value, DatetimeFormat.YMDHM.value],
+                )
+            except IndexError:
+                pass
+
+        files_server[DWDMetaColumns.DATETIME.value] = files_server[
+            DWDMetaColumns.FILENAME.value
+        ].apply(get_date_from_filename)
+
+        files_server = files_server.dropna()
+
     return files_server
 
 
@@ -107,14 +139,13 @@ def create_fileindex_radolan_grid(time_resolution: TimeResolution) -> pd.DataFra
         )
     ]
 
-    r = re.compile(f"{RADOLAN_HISTORICAL_DT_REGEX}|{RADOLAN_RECENT_DT_REGEX}")
-
-    # Require datetime of file for filtering
+    # Decode datetime of file for filtering.
+    pattern = re.compile(f"{RADOLAN_HISTORICAL_DT_REGEX}|{RADOLAN_RECENT_DT_REGEX}")
     file_index[DWDMetaColumns.DATETIME.value] = file_index[
         DWDMetaColumns.FILENAME.value
     ].apply(
         lambda filename: parse(
-            r.findall(filename)[0],
+            pattern.findall(filename)[0],
             date_formats=[DatetimeFormat.YM.value, DatetimeFormat.ymdhm.value],
         )
     )