Add mean/spread for atmos grib2 (NOAA-EMC#2482)

Adds the basic mean and spread grib2 products for the atmosphere.

The new mean/spread files are named according to the new convention,
though `pres_${grid_type}` may be changed to `pgrb_${grid_type}`. Either
way, the input filenames will need to be updated when that change is
made. For now, the mismatch means the unlettered grib files result in
`pres_`.

Resolves NOAA-EMC#2296
Refs NOAA-EMC#1522

jobs/JGLOBAL_ATMOS_ENSSTAT

@@ -0,0 +1,48 @@
+#! /usr/bin/env bash
+
+#
+# Caculate the mean, spread, and other probabilistic fields.
+#
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmos_ensstat" -c "base atmos_ensstat"
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Construct COM variables from templates
+# Input directories loop over members, so this is done downstream
+
+for grid in '0p25' '0p50' '1p00'; do
+  prod_dir="COMOUT_ATMOS_GRIB_${grid}"
+  MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
+done
+
+###############################################################
+# Run exglobal script
+"${SCRgfs}/exglobal_atmos_ensstat.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA:-NO}" = "NO" ]] && rm -rf "${DATA}"
+
+
+exit 0

jobs/rocoto/atmos_ensstat.sh

@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+## atmosphere products driver script
+## FHRLST : forecast hour list to post-process (e.g. -f001, f000, f000_f001_f002, ...)
+###############################################################
+
+# Source FV3GFS workflow modules
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+if (( status != 0 )); then exit "${status}"; fi
+
+export job="atmos_ensstat"
+export jobid="${job}.$$"
+
+###############################################################
+# shellcheck disable=SC2153,SC2001
+IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+
+#---------------------------------------------------------------
+# Execute the JJOB
+for fhr in "${fhrs[@]}"; do
+    # The analysis fhr is -001.  Performing math on negative, leading 0 integers is tricky.
+    # The negative needs to be in front of "10#", so do some regex magic to make it happen.
+    fhr="10#${fhr}"
+    fhr=${fhr//10\#-/-10\#}
+    export FORECAST_HOUR=$(( fhr ))
+    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_ENSSTAT"
+    status=$?
+    if (( status != 0 )); then exit "${status}"; fi
+done
+
+exit 0

parm/config/gefs/config.atmos_ensstat

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.atmos_ensstat ##########
+# atmosphere grib2 enstat specific
+
+echo "BEGIN: config.atmos_ensstat"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" atmos_ensstat
+
+echo "END: config.atmos_ensstat"

parm/config/gefs/config.resources

@@ -7,23 +7,6 @@
 if (( $# != 1 )); then
 
     echo "Must specify an input task argument to set resource variables!"
-    echo "argument can be any one of the following:"
-    echo "stage_ic aerosol_init"
-    echo "prep prepsnowobs prepatmiodaobs"
-    echo "atmanlinit atmanlrun atmanlfinal"
-    echo "atmensanlinit atmensanlrun atmensanlfinal"
-    echo "snowanl"
-    echo "aeroanlinit aeroanlrun aeroanlfinal"
-    echo "anal sfcanl analcalc analdiag fcst echgres"
-    echo "upp atmos_products"
-    echo "tracker genesis genesis_fsu"
-    echo "verfozn verfrad vminmon fit2obs metp arch cleanup"
-    echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
-    echo "init_chem mom6ic"
-    echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt"
-    echo "wavegempak waveawipsbulls waveawipsgridded"
-    echo "postsnd awips gempak npoess"
-    echo "ocnanalprep prepoceanobs ocnanalbmat ocnanalrun ocnanalchkpt ocnanalpost ocnanalvrfy"
     exit 1
 
 fi
@@ -232,6 +215,18 @@ case ${step} in
     export is_exclusive=True
     ;;
 
+  "atmos_ensstat")
+    export wtime_atmos_ensstat="00:30:00"
+    export npe_atmos_ensstat=6
+    export nth_atmos_ensstat=1
+    export npe_node_atmos_ensstat="${npe_atmos_ensstat}"
+    export wtime_atmos_ensstat_gfs="${wtime_atmos_ensstat}"
+    export npe_atmos_ensstat_gfs="${npe_atmos_ensstat}"
+    export nth_atmos_ensstat_gfs="${nth_atmos_ensstat}"
+    export npe_node_atmos_ensstat_gfs="${npe_node_atmos_ensstat}"
+    export is_exclusive=True
+    ;;
+
   "oceanice_products")
     export wtime_oceanice_products="00:15:00"
     export npe_oceanice_products=1

scripts/exglobal_atmos_ensstat.sh

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+fhr3=$(printf "%03d" "${FORECAST_HOUR}")
+
+if [[ -a mpmd_script ]]; then rm -Rf mpmd_script; fi
+
+{
+    for grid in '0p25' '0p50' '1p00'; do
+        echo "${USHgfs}/atmos_ensstat.sh ${grid} ${fhr3}"
+        # echo "${USHgfs}/atmos_ensstat.sh ${grid} ${fhr3} b"
+    done
+} > mpmd_script
+
+"${USHgfs}/run_mpmd.sh" mpmd_script
+err=$?
+
+exit "${err}"

sorc/link_workflow.sh

@@ -274,7 +274,8 @@ cd "${HOMEgfs}/exec" || exit 1
 
 for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.x \
   syndat_maksynrc.x syndat_qctropcy.x tocsbufr.x overgridid.x rdbfmsua.x \
-  mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x
+  mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x \
+  ensadd.x ensppf.x ensstat.x wave_stat.x
 do
   [[ -s "${utilexe}" ]] && rm -f "${utilexe}"
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/install/bin/${utilexe}" .

ush/atmos_ensstat.sh

@@ -0,0 +1,99 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+grid=${1}
+fhr3=${2}
+grid_type=${3:-''}
+
+mkdir "${grid}${grid_type}"
+cd "${grid}${grid_type}" || exit 2
+
+# Collect input grib files
+input_files=()
+for ((mem_num = 0; mem_num <= "${NMEM_ENS:-0}"; mem_num++)); do
+    mem=$(printf "%03d" "${mem_num}")
+    MEMDIR="mem${mem}" GRID="${grid}" YMD="${PDY}" HH="${cyc}" declare_from_tmpl COMIN_ATMOS_GRIB:COM_ATMOS_GRIB_GRID_TMPL
+    memfile_in="${COMIN_ATMOS_GRIB}/${RUN}.t${cyc}z.pgrb2${grid_type}.${grid}.f${fhr3}"
+
+    if [[ -r "${memfile_in}.idx" ]]; then
+        ${NLN} "${memfile_in}" "mem${mem}"
+        input_files+=("mem${mem}")
+    else
+        echo "FATAL ERROR: ${memfile_in} does not exist"
+        exit 10
+    fi
+done
+
+num_found=${#input_files[@]}
+if (( num_found != NMEM_ENS + 1 )); then
+    echo "FATAL ERROR: Only ${num_found} grib files found out of $(( NMEM_ENS + 1 )) expected members."
+    exit 10
+fi
+
+# Create namelist for ensstat
+mean_out="${RUN}.t${cyc}z.mean.pres_${grid_type}.${grid}.f${fhr3}.grib2"
+spr_out="${RUN}.t${cyc}z.spread.pres_${grid_type}.${grid}.f${fhr3}.grib2"
+
+cat << EOF > input.nml
+&namdim
+    lfdim=${lfm:-''}
+/
+
+&namens
+    nfiles=${num_found}
+    nenspost=0
+    navg_min=${NMEM_ENS}
+
+    cfopg1="${mean_out}"
+    cfopg2="${spr_out}"
+
+$(
+    for (( filenum = 1; filenum <= num_found; filenum++ )); do
+        echo "    cfipg(${filenum})=\"${input_files[$((filenum-1))]}\","
+        echo "    iskip(${filenum})=0,"
+    done
+)
+/
+EOF
+
+cat input.nml
+
+# Run ensstat
+"${EXECgfs}/ensstat.x" < input.nml
+
+export err=$?
+if (( err != 0 )) ; then
+    echo "FATAL ERROR: ensstat returned error code ${err}"
+    exit "${err}"
+fi
+
+# Send data to com and send DBN alerts
+comout_var_name="COMOUT_ATMOS_GRIB_${grid}"
+comout_path="${!comout_var_name}"
+
+for outfile in ${mean_out} ${spr_out}; do
+    if [[ ! -s ${outfile} ]]; then
+        echo "FATAL ERROR: Failed to create ${outfile}"
+        exit 20
+    fi
+
+    ${WGRIB2} -s "${outfile}" > "${outfile}.idx"
+    err=$?
+    if (( err != 0 )); then
+        echo "FATAL ERROR: Failed to create inventory file, wgrib2 returned ${err}"
+        exit "${err}"
+    fi
+
+    cpfs "${outfile}" "${comout_path}/${outfile}"
+    cpfs "${outfile}.idx" "${comout_path}/${outfile}.idx"
+
+    if [[ ${SENDDBN} == "YES" ]]; then
+        "${DBNROOT}/bin/dbn_alert" MODEL "${RUN^^}_PGB2${grid_type}_${grid}" "${job}" \
+            "${comout_path}/${outfile}"
+        "${DBNROOT}/bin/dbn_alert" MODEL "${RUN^^}_PGB2${grid_type}_${grid}" "${job}" \
+            "${comout_path}/${outfile}.idx"
+    fi
+
+done
+

workflow/applications/gefs.py

@@ -17,7 +17,7 @@ def _get_app_configs(self):
         configs = ['stage_ic', 'fcst', 'atmos_products']
 
         if self.nens > 0:
-            configs += ['efcs']
+            configs += ['efcs', 'atmos_ensstat']
 
         if self.do_wave:
             configs += ['waveinit', 'wavepostsbs', 'wavepostpnt']
@@ -52,6 +52,9 @@ def get_task_names(self):
 
         tasks += ['atmos_prod']
 
+        if self.nens > 0:
+            tasks += ['atmos_ensstat']
+
         if self.do_ocean:
             tasks += ['ocean_prod']
 

workflow/rocoto/gefs_tasks.py

@@ -233,6 +233,45 @@ def _atmosoceaniceprod(self, component: str):
 
         return task
 
+    def atmos_ensstat(self):
+
+        resources = self.get_resource('atmos_ensstat')
+
+        deps = []
+        for member in range(0, self.nmem + 1):
+            task = f'atmos_prod_mem{member:03d}_f#fhr#'
+            dep_dict = {'type': 'task', 'name': task}
+            deps.append(rocoto.add_dependency(dep_dict))
+
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        postenvars = self.envars.copy()
+        postenvar_dict = {'FHRLST': '#fhr#'}
+        for key, value in postenvar_dict.items():
+            postenvars.append(rocoto.create_envar(name=key, value=str(value)))
+
+        task_name = f'atmos_ensstat_f#fhr#'
+        task_dict = {'task_name': task_name,
+                     'resources': resources,
+                     'dependency': dependencies,
+                     'envars': postenvars,
+                     'cycledef': 'gefs',
+                     'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_ensstat.sh',
+                     'job_name': f'{self.pslot}_{task_name}_@H',
+                     'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log',
+                     'maxtries': '&MAXTRIES;'}
+
+        fhrs = self._get_forecast_hours('gefs', self._configs['atmos_ensstat'])
+        fhr_var_dict = {'fhr': ' '.join([f"{fhr:03d}" for fhr in fhrs])}
+
+        fhr_metatask_dict = {'task_name': f'atmos_ensstat',
+                             'task_dict': task_dict,
+                             'var_dict': fhr_var_dict}
+
+        task = rocoto.create_task(fhr_metatask_dict)
+
+        return task
+
     def wavepostsbs(self):
         deps = []
         for wave_grid in self._configs['wavepostsbs']['waveGRD'].split():