Fix time variable attributes for IOOS metadata

pyglider/ncprocess.py

@@ -114,7 +114,9 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml):
                 if 'units' in dss.profile_time.attrs:
                     dss.profile_time.attrs.pop('units')
                 timeunits = 'seconds since 1970-01-01T00:00:00Z'
-                dss.to_netcdf(outname, encoding={'time': {'units': timeunits}})
+                timecalendar = 'gregorian'
+                dss.to_netcdf(outname, encoding={'time': {'units': timeunits, 
+                                                          'calendar': timecalendar}})
 
                 # add traj_strlen using bare ntcdf to make IOOS happy
                 with netCDF4.Dataset(outname, 'r+') as nc:

pyglider/utils.py

@@ -67,8 +67,10 @@ def get_glider_depth(ds):
     attr = {'source': 'pressure', 'long_name': 'glider depth',
             'standard_name': 'depth', 'units': 'm',
             'comment': 'from science pressure and interpolated',
+            'instrument': 'instrument_ctd',
             'observation_type': 'calulated',
             'accuracy': '1', 'precision': '2', 'resolution': '0.02',
+            'platform': 'platform',
             'valid_min': '0', 'valid_max': '2000',
             'reference_datum': 'surface', 'positive': 'down'}
     ds['depth'].attrs = attr

tests/example-data/example-slocum/deploymentRealtime.yml

@@ -97,7 +97,7 @@ netcdf_variables:
     units:        degrees_north
     axis:         Y
     coordinates:   time depth latitude longitude
-    conversion:   nmea2deg
+#    conversion:   nmea2deg
     comment:     "Estimated between surface fixes"
     observation_type: measured
     platform:     platform
@@ -113,7 +113,7 @@ netcdf_variables:
     units:        degrees_east
     axis:         X
     coordinates:  time depth latitude longitude
-    conversion:   nmea2deg
+#    conversion:   nmea2deg
     comment:     "Estimated between surface fixes"
     observation_type: measured
     platform:     platform

tests/test_pyglider.py

@@ -67,16 +67,24 @@ def test_example_seaexplorer_metadata():
 l1tsdir = str(example_dir / 'example-slocum/L0-timeseries-test/') + '/'
 scisuffix = 'tbd'
 glidersuffix = 'sbd'
+do_direct = True
+
+if do_direct:
+    # turn *.sdb and *.tbd into timeseries netcdf files
+    outname_slocum = slocum.binary_to_timeseries(
+        binarydir, cacdir, l1tsdir, deploymentyaml_slocum, search='*.[s|t]bd',
+        profile_filt_time=20, profile_min_time=20)
+else:
+    slocum.binary_to_rawnc(
+        binarydir, rawdir_slocum, cacdir, sensorlist, deploymentyaml_slocum,
+        incremental=False, scisuffix=scisuffix, glidersuffix=glidersuffix)
+
+    slocum.merge_rawnc(rawdir_slocum, rawdir_slocum, deploymentyaml_slocum,
+                       scisuffix=scisuffix, glidersuffix=glidersuffix)
+    outname_slocum = slocum.raw_to_timeseries(
+        rawdir_slocum, l1tsdir, deploymentyaml_slocum,
+        profile_filt_time=100, profile_min_time=300)
 
-slocum.binary_to_rawnc(
-    binarydir, rawdir_slocum, cacdir, sensorlist, deploymentyaml_slocum,
-    incremental=True, scisuffix=scisuffix, glidersuffix=glidersuffix)
-
-slocum.merge_rawnc(rawdir_slocum, rawdir_slocum, deploymentyaml_slocum,
-                   scisuffix=scisuffix, glidersuffix=glidersuffix)
-outname_slocum = slocum.raw_to_timeseries(
-    rawdir_slocum, l1tsdir, deploymentyaml_slocum,
-    profile_filt_time=100, profile_min_time=300)
 output_slocum = xr.open_dataset(outname_slocum)
 # Open test data file
 test_data_slocum = xr.open_dataset(

tests/test_seaexplorer.py

@@ -2,35 +2,35 @@
 import pytest
 from pathlib import Path
 import os
-os.system('rm tests/data/realtime_rawnc/*')
 library_dir = Path(__file__).parent.parent.absolute()
-example_dir = library_dir / 'tests/example-data/'
+example_dir = str(library_dir) + '/tests/example-data/'
+data_dir = str(library_dir) + '/tests/data/'
+os.system('rm ' + data_dir + '/realtime_rawnc/*')
 
 import pyglider.seaexplorer as seaexplorer
 
-
 def test__outputname():
-    fnout, filenum = seaexplorer._outputname('tests/data/realtime_raw/sea035.12.pld1.sub.36',
-                                             'tests/data/realtime_rawnc/')
-    assert fnout == 'tests/data/realtime_rawnc/sea035.0012.pld1.sub.0036.parquet'
+    fnout, filenum = seaexplorer._outputname(data_dir + '/realtime_raw/sea035.12.pld1.sub.36',
+                                             data_dir + '/realtime_rawnc/')
+    assert fnout == data_dir + '/realtime_rawnc/sea035.0012.pld1.sub.0036.parquet'
     assert filenum == 36
 
 
 def test_raw_to_rawnc():
     # Default settings on a clean folder
-    result_default = seaexplorer.raw_to_rawnc('tests/data/realtime_raw/',
-                                              'tests/data/realtime_rawnc/', None)
+    result_default = seaexplorer.raw_to_rawnc(data_dir + '/realtime_raw/',
+                                              data_dir + '/realtime_rawnc/', None)
     # Test the reprocess flag works
-    result_reprocess = seaexplorer.raw_to_rawnc('tests/data/realtime_raw/',
-                                                'tests/data/realtime_rawnc/',
+    result_reprocess = seaexplorer.raw_to_rawnc(data_dir + '/realtime_raw/',
+                                                data_dir + '/realtime_rawnc/',
                                                 None, incremental=False)
     # Check that reprocessing not preformed by default
-    result_no_new_files = seaexplorer.raw_to_rawnc('tests/data/realtime_raw/',
-                                                   'tests/data/realtime_rawnc/',
+    result_no_new_files = seaexplorer.raw_to_rawnc(data_dir + '/realtime_raw/',
+                                                   data_dir + '/realtime_rawnc/',
                                                    None)
     # Reject all payload files with fewer than 10000 lines
-    result_strict = seaexplorer.raw_to_rawnc('tests/data/realtime_raw/',
-                                             'tests/data/realtime_rawnc/',
+    result_strict = seaexplorer.raw_to_rawnc(data_dir + '/realtime_raw/',
+                                             data_dir + '/realtime_rawnc/',
                                              None,
                                              incremental=False,
                                              min_samples_in_file=10000)
@@ -42,8 +42,8 @@ def test_raw_to_rawnc():
 
 def test__needsupdating():
     ftype = 'pld1'
-    fin = 'tests/data/realtime_raw/sea035.12.pld1.sub.36'
-    fout = 'tests/data/realtime_rawnc/sea035.0012.pld1.sub.0036.parquet'
+    fin = data_dir + '/realtime_raw/sea035.12.pld1.sub.36'
+    fout = data_dir+ '/realtime_rawnc/sea035.0012.pld1.sub.0036.parquet'
     result_badpath = seaexplorer._needsupdating(ftype, fin, 'baz')
     result_goodpath = seaexplorer._needsupdating(ftype, fin, fout)
     assert result_badpath is True
@@ -52,23 +52,23 @@ def test__needsupdating():
 
 def test_merge_rawnc():
     result_default = seaexplorer.merge_parquet(
-            'tests/data/realtime_rawnc/',
-            'tests/data/realtime_rawnc/',
-            example_dir / 'example-seaexplorer/deploymentRealtime.yml')
+            data_dir + '/realtime_rawnc/',
+            data_dir + '/realtime_rawnc/',
+            example_dir +  '/example-seaexplorer/deploymentRealtime.yml')
 
     result_sub = seaexplorer.merge_parquet(
-            'tests/data/realtime_rawnc/',
-            'tests/data/realtime_rawnc/',
-            example_dir / 'example-seaexplorer/deploymentRealtime.yml',
+            data_dir + '/realtime_rawnc/',
+            data_dir + '/realtime_rawnc/',
+            example_dir + '/example-seaexplorer/deploymentRealtime.yml',
             kind='sub')
     assert result_default is False
     assert result_sub is True
 
 
 def test__interp_gli_to_pld():
     # function should interpolate values from the glider dataset to sampling frequency of payload dataset
-    glider = pl.read_parquet('tests/data/realtime_rawnc/sea035.0012.gli.sub.0036.parquet')
-    ds = pl.read_parquet('tests/data/realtime_rawnc/sea035.0012.pld1.sub.0036.parquet')
+    glider = pl.read_parquet(data_dir + '/realtime_rawnc/sea035.0012.gli.sub.0036.parquet')
+    ds = pl.read_parquet(data_dir + '/realtime_rawnc/sea035.0012.pld1.sub.0036.parquet')
     val = glider.select("Pitch").to_numpy()[:, 0]
     pitch_interp = seaexplorer._interp_gli_to_pld(glider, ds, val, None)
     assert len(pitch_interp) == ds.shape[0]
@@ -77,14 +77,14 @@ def test__interp_gli_to_pld():
 def test_raw_to_timeseries():
     # Test default, will fail as we have sub data, not raw data
     with pytest.raises(FileNotFoundError) as missing_file_exc:
-        result_default = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/',
-                                                        'tests/data/l0-profiles/',
-                                                        example_dir / 'example-seaexplorer/deploymentRealtime.yml',
+        result_default = seaexplorer.raw_to_timeseries(data_dir + '/realtime_rawnc/',
+                                                        data_dir + '/l0-profiles/',
+                                                        example_dir + '/example-seaexplorer/deploymentRealtime.yml',
                                                         )
-    result_sub = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/',
-                                                    'tests/data/l0-profiles/',
-                                                    example_dir / 'example-seaexplorer/deploymentRealtime.yml',
+    result_sub = seaexplorer.raw_to_timeseries(data_dir + '/realtime_rawnc/',
+                                                    data_dir + '/l0-profiles/',
+                                                    example_dir + '/example-seaexplorer/deploymentRealtime.yml',
                                                     kind='sub')
     assert 'No such file or directory' in str(missing_file_exc)
-    assert result_sub == 'tests/data/l0-profiles/dfo-eva035-20190718.nc'
+    assert result_sub == data_dir + '/l0-profiles/dfo-eva035-20190718.nc'