Fix error logging when csv file is missing (#86)

* Fix error logging when csv file is missing

* Change BasicGrid to CellGrid

Fix #85

Author: wpreimes

src/ismn/components.py

@@ -19,9 +19,10 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+
 import os.path
 import sys
-from pygeogrids import BasicGrid
+from pygeogrids import CellGrid
 from typing import Union
 
 import numpy as np
@@ -571,11 +572,13 @@ def coords(self) -> (list, list):
         return lons, lats
 
     @property
-    def grid(self) -> BasicGrid:
+    def grid(self) -> CellGrid:
         """
         Get grid for all Stations in Network
         """
-        return BasicGrid(*self.coords)
+        # CellGrid is intentional
+        lons, lats = self.coords
+        return CellGrid(lons, lats, cells=np.full(len(lons), 0))
 
     @property
     def n_stations(self) -> int:
@@ -694,7 +697,7 @@ class NetworkCollection(IsmnComponent):
     ----------
     networks : OrderedDict
         Collection of network names and Networks
-    grid : BasicGrid
+    grid : CellGrid
         Grid that contains one point for each station in all networks.
     """
 
@@ -718,8 +721,11 @@ def __init__(self, networks):
             lons += net_lons
             lats += net_lats
 
-        self.grid = BasicGrid(lons, lats) if (len(lons) > 0 and
-                                              len(lats) > 0) else None
+        if (len(lons) > 0) and (len(lats) > 0):
+            # Should be CellGrid
+            self.grid: CellGrid = CellGrid(lons, lats, cells=np.full(len(lons), 0))
+        else:
+            self.grid = None
 
     def __repr__(self, indent: str = ""):
         return ",\n".join([

src/ismn/filecollection.py

@@ -78,13 +78,14 @@ def _read_station_dir(
                 root, csv[0], load_metadata=True, temp_root=temp_root)
             station_meta = static_meta_file.metadata
     except const.IsmnFileError as e:
-        logger.warning(f"Error loading static meta file: {stat_dir}/{csv[0]}. "
+        _csv = "*missing*" if len(csv) == 0 else csv[0]
+        logger.warning(f"Error loading static meta file for {stat_dir}/{_csv}. "
                        f"We will use the placeholder metadata here. "
                        f"Error traceback: {traceback.format_exc()}")
         station_meta = MetaData(
             [MetaVar(k, v) for k, v in const.CSV_META_TEMPLATE.items()]
         )
-        erroneous_files.append(os.path.join(str(root), str(csv[0])))
+        erroneous_files.append(os.path.join(str(root.path), str(stat_dir), str(_csv)))
 
     data_files = root.find_files(stat_dir, "*.stm")
 
@@ -309,7 +310,8 @@ def build_from_scratch(
 
             with open(os.path.join(log_path, log_filename), mode='a') as f:
                 f.write("\n----- Summary: Erroneous Files -----\n")
-                f.writelines(errors)
+                for e in errors:
+                    f.write(f"{e}\n")
                 f.write("\n-------------------------------------\n")
 
         ismnlog.info(info)

src/ismn/interface.py

@@ -127,7 +127,6 @@ class ISMN_Interface:
     root : IsmnRoot
         ISMN data folder or .zip access
 
-
     Properties
     ----------
     networks : OrderedDict
@@ -1021,4 +1020,54 @@ def print_climate_dict(self) -> None:
 
     def close_files(self):
         # close all open filehandlers
-        self.__file_collection.close()
+        self.__file_collection.close()
+
+if __name__ == '__main__':
+    ds = ISMN_Interface("/home/wpreimes/shares/climers/Datapool/ISMN/01_raw/FullArchive_20240314")
+
+    idx_good = ds.metadata[
+        np.isin(ds.metadata[('frm_class', 'val')].values, ['representative', 'very representative'])].index
+
+    frm = ds.subset_from_ids(idx_good)
+
+    fig, ax = plt.subplots(figsize=(6, 4), subplot_kw={'projection': ccrs.PlateCarree()})
+    frm.plot_station_locations('soil_moisture', 0, 0.1,
+                              check_only_sensor_depth_from=False,
+                               borders=False, ax=ax, text_scalefactor=1.6)
+
+    fig.savefig("/tmp/frm.png", dpi=300)
+
+    fig, ax = plt.subplots(figsize=(12, 8), subplot_kw={'projection': ccrs.PlateCarree()})
+
+    frm.plot_station_locations('soil_moisture', 0, 0.1,
+                              check_only_sensor_depth_from=False,
+                               extent=[-179, 189, -60, 90],
+                               borders=False, ax=ax, markersize=15,
+                               text_scalefactor=2, stats_text=False)
+
+    fig.savefig("/tmp/frms.png", dpi=300)
+    plt.close(fig)
+
+
+    fig, ax = plt.subplots(figsize=(6, 4), subplot_kw={'projection': ccrs.PlateCarree()})
+
+    frm.plot_station_locations('soil_moisture', 0, 0.1,
+                               check_only_sensor_depth_from=False,
+                               extent=[-128, -64, 25, 50],
+                               borders=True, ax=ax, stats_text=False,
+                               markersize=11, text_scalefactor=0.7)
+
+    fig.savefig("/tmp/frms_usa.png", dpi=300)
+
+
+    plt.close(fig)
+
+    fig, ax = plt.subplots(figsize=(6, 4), subplot_kw={'projection': ccrs.PlateCarree()})
+
+    frm.plot_station_locations('soil_moisture', 0, 0.1,
+                               check_only_sensor_depth_from=False,
+                               extent=[-10, 33, 35, 70],
+                               borders=True, ax=ax, stats_text=False,
+                               markersize=11, text_scalefactor=0.7)
+
+    fig.savefig("/tmp/frms_eu.png", dpi=300)

tests/test_components.py

@@ -32,7 +32,7 @@
 from datetime import datetime, timedelta
 
 from ismn.components import NetworkCollection, Network, Station, Sensor, Depth
-from pygeogrids.grids import BasicGrid
+from pygeogrids.grids import CellGrid
 from tempfile import TemporaryDirectory
 import json
 
@@ -119,7 +119,7 @@ def test_add_station(self):
         assert self.network.stations[name].lat == 0
         assert self.network.stations[name].elev == 0
 
-        assert self.network.grid == BasicGrid([0], [0])
+        assert self.network.grid == CellGrid([0], [0], [0])
 
     def test_remove_station(self):
         """