Enhance bufr2ioda converter error checking, update ATM yamls to jinja2 (

#937)

parm/atm/berror/hybvar_bump.yaml → parm/atm/berror/hybvar_bump.yaml.j2

@@ -10,7 +10,7 @@ components:
       active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr]
       bump:
         io:
-          data directory: &staticb_dir $(DATA)/berror
+          data directory: &staticb_dir '{{ DATA }}/berror'
         drivers:
           multivariate strategy: specific_univariate
           read local nicas: true
@@ -105,7 +105,7 @@ components:
         filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
         filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
       pattern: '%mem%'
-      nmembers: $(NMEM_ENS)
+      nmembers: {{ NMEM_ENS }}
       zero padding: 3
 #    localization:
 #      localization method: SABER

parm/atm/berror/hybvar_gsibec.yaml → parm/atm/berror/hybvar_gsibec.yaml.j2

@@ -5,12 +5,12 @@ components:
     saber central block:
       saber block name: gsi covariance
       read:
-        gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4
-        gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4
-#       gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77
-        gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml
-        processor layout x direction: &layout_gsib_x $(layout_gsib_x)
-        processor layout y direction: &layout_gsib_y $(layout_gsib_y)
+        gsi akbk: &akbk '{{ DATA }}/fv3jedi/akbk.nc4'
+        gsi error covariance file: &gsiberr '{{ DATA }}/berror/gsi-coeffs-gfs-global.nc4'
+#       gsi error covariance file: &gsiberr '{{ DATA }}/berror/global_berror.f77'
+        gsi berror namelist file: &gsibnml '{{ DATA }}/berror/gfs_gsi_global.nml'
+        processor layout x direction: &layout_gsib_x {{ layout_gsib_x }}
+        processor layout y direction: &layout_gsib_y {{ layout_gsib_y }}
         debugging mode: false
     saber outer blocks:
     - saber block name: gsi interpolation to model grid
@@ -42,7 +42,7 @@ components:
         filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
         filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
       pattern: '%mem%'
-      nmembers: $(NMEM_ENS)
+      nmembers: {{ NMEM_ENS }}
       zero padding: 3
     localization:
       localization method: SABER
@@ -61,6 +61,8 @@ components:
             level for 2d variables: last
           nicas:
             resolution: 6
+            min effective resolution: 0.2  # Note: this is very much not recommended (resolution too low), only
+                                           # used because covariance quality is not important for CI tests
             explicit length-scales: true
             horizontal length-scale:
             - groups:

parm/atm/berror/staticb_bump.yaml → parm/atm/berror/staticb_bump.yaml.j2

@@ -8,7 +8,7 @@ saber blocks:
   active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr]
   bump:
     io:
-      data directory: &staticb_dir $(DATA)/berror
+      data directory: &staticb_dir '{{ DATA }}/berror'
     drivers:
       multivariate strategy: specific_univariate
       read local nicas: true

parm/atm/berror/staticb_gsibec.yaml → parm/atm/berror/staticb_gsibec.yaml.j2

@@ -3,12 +3,12 @@ full inverse: true
 saber central block:
   saber block name: gsi covariance
   read:
-    gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4
-    gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4
-#   gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77
-    gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml
-    processor layout x direction: &layout_gsib_x $(layout_gsib_x)
-    processor layout y direction: &layout_gsib_y $(layout_gsib_y)
+    gsi akbk: &akbk '{{ DATA }}/fv3jedi/akbk.nc4'
+    gsi error covariance file: &gsiberr '{{ DATA }}/berror/gsi-coeffs-gfs-global.nc4'
+#   gsi error covariance file: &gsiberr '{{ DATA }}/berror/global_berror.f77'
+    gsi berror namelist file: &gsibnml '{{ DATA }}/berror/gfs_gsi_global.nml'
+    processor layout x direction: &layout_gsib_x {{ layout_gsib_x }}
+    processor layout y direction: &layout_gsib_y {{ layout_gsib_y }}
     debugging mode: false
 saber outer blocks:
 - saber block name: gsi interpolation to model grid

parm/atm/hofx/hofx4d.yaml → parm/atm/hofx/hofx4d.yaml.j2

@@ -1,21 +1,21 @@
 time window:
   begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-  length: $(ATM_WINDOW_LENGTH)
-forecast length: $(ATM_WINDOW_LENGTH)
+  length: '{{ ATM_WINDOW_LENGTH }}'
+forecast length: '{{ ATM_WINDOW_LENGTH }}'
 geometry:
   fms initialization:
-    namelist filename: $(DATA)/fv3jedi/fmsmpp.nml
-    field table filename: $(DATA)/fv3jedi/field_table
-  akbk: $(DATA)/fv3jedi/akbk.nc4
+    namelist filename: '{{ DATA }}/fv3jedi/fmsmpp.nml'
+    field table filename: '{{ DATA }}/fv3jedi/field_table'
+  akbk: '{{ DATA }}/fv3jedi/akbk.nc4'
   layout:
-  - $(layout_x)
-  - $(layout_y)
-  npx: $(npx_ges)
-  npy: $(npy_ges)
-  npz: $(npz_ges)
-  field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml
+  - {{ layout_x }}
+  - {{ layout_y }}
+  npx: {{ npx_ges }}
+  npy: {{ npy_ges }}
+  npz: {{ npz_ges }}
+  field metadata override: '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'
 initial condition:
-  datapath: $(DATA)/bkg
+  datapath: '{{ DATA }}/bkg'
   filetype: fms restart
   datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
   filename_core: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_core.res.nc'
@@ -28,7 +28,7 @@ initial condition:
                     u_srf,v_srf,f10m]
 model:
   name: PSEUDO
-  datapath: $(DATA)/bkg
+  datapath: '{{ DATA }}/bkg'
   filetype: fms restart
   datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
   filename is datetime templated: true
@@ -37,8 +37,12 @@ model:
   filename_sfcd: '%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc'
   filename_sfcw: '%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc'
   filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res'
-  tstep: $(BKG_TSTEP)
+  tstep: {{ BKG_TSTEP }}
   model variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis,
                     slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                     u_srf,v_srf,f10m]
-observations: !INC ${OBS_LIST}
+observations:
+{% filter indent(width=2) %}
+{% set atm_obs_list = 'OBS_LIST' | getenv('atm/obs/lists/gdas_prototype.yaml.j2') %}
+{% include atm_obs_list %}
+{% endfilter %}

parm/atm/hofx/hofx_nomodel.yaml → parm/atm/hofx/hofx_nomodel.yaml.j2

@@ -1,20 +1,20 @@
 time window:
   begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-  length: $(ATM_WINDOW_LENGTH)
+  length: '{{ ATM_WINDOW_LENGTH }}'
 geometry:
   fms initialization:
-    namelist filename: $(DATA)/fv3jedi/fmsmpp.nml
-    field table filename: $(DATA)/fv3jedi/field_table
-  akbk: $(DATA)/fv3jedi/akbk.nc4
+    namelist filename: '{{ DATA }}/fv3jedi/fmsmpp.nml'
+    field table filename: '{{ DATA }}/fv3jedi/field_table'
+  akbk: '{{ DATA }}/fv3jedi/akbk.nc4'
   layout:
-  - $(layout_x)
-  - $(layout_y)
-  npx: $(npx_ges)
-  npy: $(npy_ges)
-  npz: $(npz_ges)
-  field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml
+  - {{ layout_x }}
+  - {{ layout_y }}
+  npx: {{ npx_ges }}
+  npy: {{ npy_ges }}
+  npz: {{ npz_ges }}
+  field metadata override: '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'
 state:
-  datapath: $(DATA)/bkg
+  datapath: '{{ DATA }}/bkg'
   filetype: fms restart
   datetime: '{{ current_cycle | to_isotime }}'
   filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc'
@@ -25,4 +25,8 @@ state:
   state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis,
                     slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                     u_srf,v_srf,f10m]
-observations: !INC ${OBS_LIST}
+observations:
+{% filter indent(width=2) %}
+{% set atm_obs_list = 'OBS_LIST' | getenv('atm/obs/lists/gdas_prototype.yaml.j2') %}
+{% include atm_obs_list %}
+{% endfilter %}

parm/atm/hofx/hofx_ufotest.yaml.j2

@@ -0,0 +1,8 @@
+time window:
+  begin: '{{ ATM_WINDOW_BEGIN }}'
+  end: '{{ ATM_WINDOW_END }}'
+observations:
+{% filter indent(width=2) %}
+{% set atm_obs_list = 'OBS_LIST' | getenv('atm/obs/lists/gdas_prototype.yaml.j2') %}
+{% include atm_obs_list %}
+{% endfilter %}

parm/atm/lgetkf/lgetkf.yaml → parm/atm/lgetkf/lgetkf.yaml.j2

@@ -4,15 +4,15 @@ geometry:
     field table filename: ./fv3jedi/field_table
   akbk: ./fv3jedi/akbk.nc4
   layout:
-  - $(layout_x)
-  - $(layout_y)
-  npx: $(npx_ges)
-  npy: $(npy_ges)
-  npz: $(npz_ges)
+  - {{ layout_x }}
+  - {{ layout_y }}
+  npx: {{ npx_ges }}
+  npy: {{ npy_ges }}
+  npz: {{ npz_ges }}
   field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml
 time window:
   begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-  length: $(ATM_WINDOW_LENGTH)
+  length: '{{ ATM_WINDOW_LENGTH }}'
 increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr]
 
 background:
@@ -31,10 +31,14 @@ background:
        filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
        filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
      pattern: '%mem%'
-     nmembers: $(NMEM_ENS)
+     nmembers: {{ NMEM_ENS }}
      zero padding: 3
 
-observations: !INC ${OBS_LIST}
+observations:
+{% filter indent(width=2) %}
+{% set atm_obs_list = 'OBS_LIST' | getenv('atm/obs/lists/lgetkf_prototype.yaml.j2') %}
+{% include atm_obs_list %}
+{% endfilter %}
 
 driver:
   update obs config with geometry info: true