copy subsampled marine BUFR to DMPDIR (#878)

This PR makes a copy of the subsampled marine BUFR to DMPDIR as part of
the marine obs prep task

closes #804
closes #849

parm/soca/obs/config/insitu_profile_bathy.yaml

@@ -0,0 +1,22 @@
+obs space:
+  name: insitu_profile_bathy
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_bathy.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/insitu_profile_bathy.${PDY}${cyc}.nc4
+  simulated variables: [waterTemperature, salinity]
+  observed variables: [waterTemperature, salinity]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: Composite
+  components:
+  - name: InsituTemperature
+    variables:
+    - name: waterTemperature
+obs error:
+  covariance model: diagonal

parm/soca/obs/config/insitu_profile_tesac.yaml

@@ -0,0 +1,22 @@
+obs space:
+  name: insitu_profile_tesac
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_tesac.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/insitu_profile_tesac.${PDY}${cyc}.nc4
+  simulated variables: [waterTemperature, salinity]
+  observed variables: [waterTemperature, salinity]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: Composite
+  components:
+  - name: InsituTemperature
+    variables:
+    - name: waterTemperature
+obs error:
+  covariance model: diagonal

parm/soca/obs/config/insitu_surface_trkob.yaml

@@ -0,0 +1,18 @@
+obs space:
+  name: insitu_surface_trkob
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_surface_trkob.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/insitu_surface_trkob.${PDY}${cyc}.nc4
+  simulated variables: [seaSurfaceTemperature]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: Identity
+  observation alias file: obsop_name_map.yaml
+obs error:
+  covariance model: diagonal

parm/soca/obs/obs_list.yaml

@@ -15,3 +15,6 @@ observers:
 # Ice concentration
 - !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml
 - !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml
+- !INC ${OBS_YAML_DIR}/insitu_profile_bathy.yaml
+- !INC ${OBS_YAML_DIR}/insitu_profile_tesac.yaml
+- !INC ${OBS_YAML_DIR}/insitu_surface_trkob.yaml

parm/soca/obsprep/obsprep_config.yaml

@@ -149,3 +149,21 @@ observations:
     binning:
       stride: 15
       min number of obs: 10
+- obs space:
+    name: insitu_profile_bathy
+    provider: GTS
+    dmpdir subdir: atmos
+    output file: '*.bathy_profile.ioda.nc'
+    dmpdir regex: 'bathy.*.dcom_subsampled'
+- obs space:
+    name: insitu_profile_tesac
+    provider: GTS
+    dmpdir subdir: atmos
+    output file: '*.tesac_profile.ioda.nc'
+    dmpdir regex: 'tesac.*.dcom_subsampled'
+- obs space:
+    name: insitu_surface_trkob
+    provider: GTS
+    dmpdir subdir: atmos
+    output file: '*.trkob_surface.ioda.nc'
+    dmpdir regex: 'trkob.*.dcom_subsampled'

scripts/exglobal_prep_ocean_obs.py

@@ -68,7 +68,6 @@ def run_netcdf_to_ioda(obsspace_to_convert):
 
             if obsprepSpaceName == obs_space_name:
                 logger.info(f"obsprepSpaceName: {obs_space_name}")
-
                 pdyDatetime = datetime.strptime(PDY + cyc, '%Y%m%d%H')
                 cycles = []
 
@@ -95,15 +94,19 @@ def run_netcdf_to_ioda(obsspace_to_convert):
                 outputFilename = f"gdas.t{cyc}z.{obs_space_name}.{PDY}{cyc}.nc4"
                 obsprepSpace['output file'] = outputFilename
 
-                iodaYamlFilename = obsprepSpaceName + '2ioda.yaml'
-                save_as_yaml(obsprepSpace, iodaYamlFilename)
+                # Skip in situ IODA conversion for now
+                if obsprepSpaceName.split('_')[0] == 'insitu':
+                    logger.info("Skipping insitu conversion for now")
+                else:
+                    iodaYamlFilename = obsprepSpaceName + '2ioda.yaml'
+                    save_as_yaml(obsprepSpace, iodaYamlFilename)
 
-                files_to_save.append([obsprepSpace['output file'],
-                                      os.path.join(COMOUT_OBS, obsprepSpace['output file'])])
-                files_to_save.append([iodaYamlFilename,
-                                      os.path.join(COMOUT_OBS, iodaYamlFilename)])
+                    files_to_save.append([obsprepSpace['output file'],
+                                          os.path.join(COMOUT_OBS, obsprepSpace['output file'])])
+                    files_to_save.append([iodaYamlFilename,
+                                          os.path.join(COMOUT_OBS, iodaYamlFilename)])
 
-                obsspaces_to_convert.append((obs_space_name, iodaYamlFilename))
+                    obsspaces_to_convert.append((obs_space_name, iodaYamlFilename))
 
 except TypeError:
     logger.critical("Ill-formed OBS_YAML or OBSPREP_YAML file, exiting")

test/soca/gw/setup_obsprep.sh

@@ -3,8 +3,6 @@ set -ex
 
 # working directory as set in cmake assumed to be ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep
 # which is the soca ctest's fake dmpdir
-
-
 # Ensure project source directory is provided as argument
 if [ "$#" -ne 1 ]; then
     echo "Usage: $0 <project_source_dir>"
@@ -13,31 +11,44 @@ fi
 
 project_source_dir="$1"
 testdatadir="${project_source_dir}/test/soca/testdata"
+testdatadir_bufr="${project_source_dir}/build/gdas/test/testdata"
 
-#clean up previous attempts
+# Clean up previous attempts
 rm -rf gdas.20180414  gdas.20180415
 
 # Define PDYs, cycs, and obstypes
 PDYs=("20180414" "20180415")
 cycs=("00" "06" "12" "18")
-obstypes=("SSS" "adt" "icec" "sst")
+obstypes=("insitu" "sss" "adt" "icec" "sst")
 
-# Convert cdl files into nc for all cycles and obstypes
+# Copy marine obs (in situ BUFR and satellite retrieved netCDF files) to DMPDIR
 for PDY in "${PDYs[@]}"; do
     PDYdir="gdas.${PDY}"
     for cyc in "${cycs[@]}"; do
         for obstype in "${obstypes[@]}"; do
-            fullsubdir="$PDYdir/$cyc/ocean/$obstype"
-            mkdir -p "$fullsubdir"
-
-            indir="${testdatadir}/${fullsubdir}"
-            for file in "$indir"/*.cdl; do
-                if [ -f "$file" ]; then
-                    filename=$(basename -- "$file")
-                    filename_noext="${filename%.cdl}"
-                    ncgen -o "$fullsubdir/${filename_noext}.nc" "$file"
-                fi
-            done
+            if [ $obstype != "insitu" ]; then
+                fullsubdir="$PDYdir/$cyc/ocean/$obstype"
+                mkdir -p "$fullsubdir"
+
+                # Convert cdl files into nc for all cycles and obstypes
+                indir="${testdatadir}/${fullsubdir}"
+                for file in "$indir"/*.cdl; do
+                    if [ -f "$file" ]; then
+                        filename=$(basename -- "$file")
+                        filename_noext="${filename%.cdl}"
+                        ncgen -o "$fullsubdir/${filename_noext}.nc" "$file"
+                    fi
+                done
+            else
+                # Copy subsampled monthly in situ BUFR
+                # Example filename: tesac.201804.dcom_subsampled
+                # TODO: SP to replace these with daily and monthly subsampled with proper dates 
+                fullsubdir="$PDYdir/$cyc/atmos"
+                mkdir -p "$fullsubdir"
+                for file in "$testdatadir_bufr/*.${PDY:0:6}.dcom_subsampled"; do
+                    cp -p $file $fullsubdir
+                done
+            fi
         done
     done
 done