From 1ccc9896b361f2aaef8e6e7592a06ae4cfb7c491 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Mon, 5 Feb 2024 14:16:07 -0500 Subject: [PATCH] Remove EnKF forecast groups (#2280) Removes the grouping of EnKF forecasts so each job only runs one forecast. Member and MEMDIR are now set at the workflow manager (rocoto) level. This change makes much of the system simpler (especially dependencies) and allows the elimination of the separate efcs scripts. Metatask names of updated jobs have been updated to make them a little less opaque by using the same name name as its constituent tasks (e.g. the forecast metatask is named `enkgdasfcst`, not `enkfgdasefcs`). Metatasks that weren't updated retain the same names as before for now. Resolves #2254 --- jobs/JGDAS_ENKF_FCST | 84 ------------ jobs/JGLOBAL_FORECAST | 60 ++++----- jobs/rocoto/efcs.sh | 25 ---- parm/config/gefs/config.efcs | 35 +++-- parm/config/gefs/config.fcst | 6 + parm/config/gfs/config.efcs | 55 ++++++-- parm/config/gfs/config.fcst | 18 +++ parm/config/gfs/config.resources | 10 +- scripts/exgdas_enkf_fcst.sh | 225 ------------------------------- ush/forecast_predet.sh | 5 +- ush/hpssarch_gen.sh | 4 +- workflow/rocoto/gefs_tasks.py | 21 +-- workflow/rocoto/gfs_tasks.py | 48 +++---- workflow/rocoto/tasks.py | 7 +- workflow/setup_expt.py | 4 +- 15 files changed, 160 insertions(+), 447 deletions(-) delete mode 100755 jobs/JGDAS_ENKF_FCST delete mode 100755 jobs/rocoto/efcs.sh delete mode 100755 scripts/exgdas_enkf_fcst.sh diff --git a/jobs/JGDAS_ENKF_FCST b/jobs/JGDAS_ENKF_FCST deleted file mode 100755 index 53408df8cf..0000000000 --- a/jobs/JGDAS_ENKF_FCST +++ /dev/null @@ -1,84 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "efcs" -c "base fcst efcs" - - -############################################## -# Set variables used in the script -############################################## -export CDUMP=${RUN/enkf} - -############################################## -# Begin JOB SPECIFIC work -############################################## - -export CASE=${CASE_ENS} - -YMD=${PDY} HH=${cyc} generate_com -rx COM_TOP - - -# Forecast length for EnKF forecast -export FHMIN=${FHMIN_ENKF} -export FHOUT=${FHOUT_ENKF} -export FHMAX=${FHMAX_ENKF} - -# Get ENSBEG/ENSEND from ENSGRP and NMEM_EFCSGRP -if [[ $CDUMP == "gfs" ]]; then - export NMEM_EFCSGRP=${NMEM_EFCSGRP_GFS:-${NMEM_EFCSGRP:-1}} -fi -export ENSEND=$((NMEM_EFCSGRP * 10#${ENSGRP})) -export ENSBEG=$((ENSEND - NMEM_EFCSGRP + 1)) - -if [[ ${DO_WAVE} == "YES" ]]; then - declare -rx RUNwave="${RUN}wave" -fi - -############################################################### -# Run relevant script - -${ENKFFCSTSH:-${SCRgfs}/exgdas_enkf_fcst.sh} -status=$? -[[ ${status} -ne 0 ]] && exit ${status} - - -# Double check the status of members in ENSGRP -EFCSGRP="${COM_TOP}/efcs.grp${ENSGRP}" -npass=0 -if [ -f ${EFCSGRP} ]; then - npass=$(grep "PASS" ${EFCSGRP} | wc -l) -fi -echo "${npass}/${NMEM_EFCSGRP} members successfull in efcs.grp${ENSGRP}" -if [ ${npass} -ne ${NMEM_EFCSGRP} ]; then - echo "FATAL ERROR: Failed members in group ${ENSGRP}, ABORT!" - cat ${EFCSGRP} - exit 99 -fi - - -############################################## -# Send Alerts -############################################## -if [ ${SENDDBN} = YES ] ; then - ${DBNROOT}/bin/dbn_alert MODEL ENKF1_MSC_fcsstat ${job} ${EFCSGRP} -fi - - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} -fi - -########################################## -# Remove the Temporary working directory -########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} - -exit 0 diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST index b2825af54f..4e2f49ca37 100755 --- a/jobs/JGLOBAL_FORECAST +++ b/jobs/JGLOBAL_FORECAST @@ -1,40 +1,19 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "fcst" -c "base fcst" - -############################################## -# Set variables used in the script -############################################## -export CDUMP=${RUN/enkf} +if (( ${ENSMEM:-0} > 0 )); then + source "${HOMEgfs}/ush/jjob_header.sh" -e "efcs" -c "base fcst efcs" +else + source "${HOMEgfs}/ush/jjob_header.sh" -e "fcst" -c "base fcst" +fi ############################################## # Begin JOB SPECIFIC work ############################################## # Restart conditions for GFS cycle come from GDAS -rCDUMP=${CDUMP} -[[ ${CDUMP} = "gfs" ]] && export rCDUMP="gdas" - -# Forecast length for GFS forecast -case ${RUN} in - *gfs | *gefs) - # shellcheck disable=SC2153 - export FHMAX=${FHMAX_GFS} - # shellcheck disable=SC2153 - export FHOUT=${FHOUT_GFS} - export FHMAX_HF=${FHMAX_HF_GFS} - export FHOUT_HF=${FHOUT_HF_GFS} - ;; - *gdas) - export FHMAX_HF=0 - export FHOUT_HF=0 - ;; - *) - echo "FATAL ERROR: Unsupported RUN '${RUN}'" - exit 1 -esac - +rCDUMP=${RUN} +[[ ${RUN} == "gfs" ]] && export rCDUMP="gdas" # Ignore possible spelling error (nothing is misspelled) # shellcheck disable=SC2153 @@ -78,11 +57,21 @@ fi ############################################################### # Run relevant exglobal script - +############################################################### ${FORECASTSH:-${SCRgfs}/exglobal_forecast.sh} status=$? -[[ ${status} -ne 0 ]] && exit ${status} - +[[ ${status} -ne 0 ]] && exit "${status}" + +# Send DBN alerts for EnKF +# TODO: Should these be in post manager instead? +if [[ "${RUN}" =~ "enkf" ]] && [[ "${SENDDBN}" = YES ]]; then + for (( fhr = FHOUT; fhr <= FHMAX; fhr + FHOUT )); do + if (( fhr % 3 == 0 )); then + fhr3=$(printf %03i "${fhr}") + "${DBNROOT}/bin/dbn_alert" MODEL GFS_ENKF "${job}" "${COM_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${fhr3}.nc" + fi + done +fi ############################################## # End JOB SPECIFIC work @@ -91,15 +80,14 @@ status=$? ############################################## # Final processing ############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" fi ########################################## # Remove the Temporary working directory ########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} - +cd "${DATAROOT}" || true +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/jobs/rocoto/efcs.sh b/jobs/rocoto/efcs.sh deleted file mode 100755 index c5667cb970..0000000000 --- a/jobs/rocoto/efcs.sh +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -# Source FV3GFS workflow modules -# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2 -source "${HOMEgfs}/ush/detect_machine.sh" -if [[ "${MACHINE_ID}" == "wcoss2" ]]; then - . ${HOMEgfs}/ush/load_ufswm_modules.sh -else - . ${HOMEgfs}/ush/load_fv3gfs_modules.sh -fi -status=$? -[[ ${status} -ne 0 ]] && exit ${status} - -export job="efcs" -export jobid="${job}.$$" - -############################################################### -# Execute the JJOB -"${HOMEgfs}/jobs/JGDAS_ENKF_FCST" -status=$? - -exit ${status} diff --git a/parm/config/gefs/config.efcs b/parm/config/gefs/config.efcs index 9593408848..a6f34818d7 100644 --- a/parm/config/gefs/config.efcs +++ b/parm/config/gefs/config.efcs @@ -5,14 +5,16 @@ echo "BEGIN: config.efcs" -# Turn off components in ensemble via _ENKF, or use setting from deterministic -export DO_AERO=${DO_AERO_ENKF:-${DO_AERO:-"NO"}} -export DO_OCN=${DO_OCN_ENKF:-${DO_OCN:-"NO"}} -export DO_ICE=${DO_ICE_ENKF:-${DO_ICE:-"NO"}} -export DO_WAVE=${DO_WAVE_ENKF:-${DO_WAVE:-"NO"}} +# Turn off components in ensemble +# export DO_AERO="NO" +# export DO_OCN="NO" +# export DO_ICE="NO" +# export DO_WAVE="NO" + +export CASE="${CASE_ENS}" # Source model specific information that is resolution dependent -string="--fv3 ${CASE_ENS}" +string="--fv3 ${CASE}" # Ocean/Ice/Waves ensemble configurations are identical to deterministic member [[ "${DO_OCN}" == "YES" ]] && string="${string} --mom6 ${OCNRES}" [[ "${DO_ICE}" == "YES" ]] && string="${string} --cice6 ${ICERES}" @@ -24,17 +26,22 @@ source "${EXPDIR}/config.ufs" ${string} # Get task specific resources source "${EXPDIR}/config.resources" efcs +# nggps_diag_nml +export FHOUT=${FHOUT_ENKF:-3} +if [[ "${RUN}" == "enkfgfs" ]]; then + export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT}} +fi + +# model_configure +export FHMAX=${FHMAX_ENKF:-9} +if [[ "${RUN}" == "enkfgfs" ]]; then + export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX}} +fi + # Use serial I/O for ensemble (lustre?) export OUTPUT_FILETYPE_ATM="netcdf" export OUTPUT_FILETYPE_SFC="netcdf" -# Number of enkf members per fcst job -export NMEM_EFCSGRP=1 -export RERUN_EFCSGRP="NO" - -# Turn off inline UPP for EnKF forecast -export WRITE_DOPOST=".true." - # Stochastic physics parameters (only for ensemble forecasts) export DO_SKEB="YES" export SKEB=0.3 @@ -54,6 +61,6 @@ export SPPT_LSCALE=500000. export SPPT_LOGIT=".true." export SPPT_SFCLIMIT=".true." -export restart_interval=${restart_interval_gfs} +export restart_interval="${restart_interval_gfs}" echo "END: config.efcs" diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 6f4a11f7c4..6a2a852e0b 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -21,6 +21,12 @@ string="--fv3 ${CASE}" # shellcheck disable=SC2086 source "${EXPDIR}/config.ufs" ${string} +# shellcheck disable=SC2153 +export FHMAX=${FHMAX_GFS} +# shellcheck disable=SC2153 +export FHOUT=${FHOUT_GFS} +export FHMAX_HF=${FHMAX_HF_GFS} +export FHOUT_HF=${FHOUT_HF_GFS} # Get task specific resources source "${EXPDIR}/config.resources" fcst diff --git a/parm/config/gfs/config.efcs b/parm/config/gfs/config.efcs index 283ec3ab7e..7a48feb4ce 100644 --- a/parm/config/gfs/config.efcs +++ b/parm/config/gfs/config.efcs @@ -5,14 +5,16 @@ echo "BEGIN: config.efcs" -# Turn off components in ensemble via _ENKF, or use setting from deterministic -export DO_AERO=${DO_AERO_ENKF:-${DO_AERO:-"NO"}} -export DO_OCN=${DO_OCN_ENKF:-${DO_OCN:-"NO"}} -export DO_ICE=${DO_ICE_ENKF:-${DO_ICE:-"NO"}} -export DO_WAVE=${DO_WAVE_ENKF:-${DO_WAVE:-"NO"}} +# Turn off components in ensemble +# export DO_AERO="NO" +# export DO_OCN="NO" +# export DO_ICE="NO" +export DO_WAVE="NO" + +export CASE="${CASE_ENS}" # Source model specific information that is resolution dependent -string="--fv3 ${CASE_ENS}" +string="--fv3 ${CASE}" # Ocean/Ice/Waves ensemble configurations are identical to deterministic member [[ "${DO_OCN}" == "YES" ]] && string="${string} --mom6 ${OCNRES}" [[ "${DO_ICE}" == "YES" ]] && string="${string} --cice6 ${ICERES}" @@ -25,15 +27,23 @@ source "${EXPDIR}/config.ufs" ${string} # Get task specific resources . "${EXPDIR}/config.resources" efcs +# nggps_diag_nml +export FHOUT=${FHOUT_ENKF:-3} +if [[ ${RUN} == "enkfgfs" ]]; then + export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT}} +fi + +# model_configure +export FHMIN=${FHMIN_ENKF:-3} +export FHMAX=${FHMAX_ENKF:-9} +if [[ ${RUN} == "enkfgfs" ]]; then + export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX}} +fi + # Use serial I/O for ensemble (lustre?) export OUTPUT_FILETYPE_ATM="netcdf" export OUTPUT_FILETYPE_SFC="netcdf" -# Number of enkf members per fcst job -export NMEM_EFCSGRP=2 -export NMEM_EFCSGRP_GFS=1 -export RERUN_EFCSGRP="NO" - # Turn off inline UPP for EnKF forecast export WRITE_DOPOST=".false." @@ -56,14 +66,33 @@ export SPPT_LSCALE=500000. export SPPT_LOGIT=".true." export SPPT_SFCLIMIT=".true." -if [[ "${QUILTING}" = ".true." ]] && [[ "${OUTPUT_GRID}" = "gaussian_grid" ]]; then +if [[ "${QUILTING}" == ".true." ]] && [[ "${OUTPUT_GRID}" == "gaussian_grid" ]]; then export DIAG_TABLE="${HOMEgfs}/parm/ufs/fv3/diag_table_da" else export DIAG_TABLE="${HOMEgfs}/parm/ufs/fv3/diag_table_da_orig" fi +# Model config option for Ensemble +# export TYPE=nh # choices: nh, hydro +# export MONO=non-mono # choices: mono, non-mono + +# gfs_physics_nml +export FHSWR=3600. +export FHLWR=3600. +export IEMS=1 +export ISOL=2 +export ICO2=2 +export dspheat=".true." +export shal_cnv=".true." +export FHZER=6 + +# Set PREFIX_ATMINC to r when recentering on +if [[ ${RECENTER_ENKF:-"YES"} == "YES" ]]; then + export PREFIX_ATMINC="r" +fi + # For IAU, write restarts at beginning of window also -if [[ "${DOIAU_ENKF:-}" = "YES" ]]; then +if [[ "${DOIAU_ENKF:-}" == "YES" ]]; then export restart_interval="3" else export restart_interval="6" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 6c4ce639f8..d2e2664e9c 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -21,6 +21,24 @@ string="--fv3 ${CASE}" # shellcheck disable=SC2086 source "${EXPDIR}/config.ufs" ${string} +# Forecast length for GFS forecast +case ${RUN} in + *gfs) + # shellcheck disable=SC2153 + export FHMAX=${FHMAX_GFS} + # shellcheck disable=SC2153 + export FHOUT=${FHOUT_GFS} + export FHMAX_HF=${FHMAX_HF_GFS} + export FHOUT_HF=${FHOUT_HF_GFS} + ;; + *gdas) + export FHMAX_HF=0 + export FHOUT_HF=0 + ;; + *) + echo "FATAL ERROR: Unsupported RUN '${RUN}'" + exit 1 +esac # Get task specific resources source "${EXPDIR}/config.resources" fcst diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 36d8e788c3..1dcf757de2 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -623,15 +623,15 @@ case ${step} in case "${CASE}" in "C48" | "C96" | "C192") - declare -x "wtime_${step}"="00:30:00" + declare -x "wtime_${step}"="00:15:00" declare -x "wtime_${step}_gfs"="03:00:00" ;; "C384") - declare -x "wtime_${step}"="00:20:00" + declare -x "wtime_${step}"="00:10:00" declare -x "wtime_${step}_gfs"="06:00:00" ;; "C768" | "C1152") - declare -x "wtime_${step}"="01:00:00" + declare -x "wtime_${step}"="00:30:00" declare -x "wtime_${step}_gfs"="06:00:00" ;; *) @@ -809,7 +809,7 @@ case ${step} in ;; "cleanup") - export wtime_cleanup="01:00:00" + export wtime_cleanup="00:15:00" export npe_cleanup=1 export npe_node_cleanup=1 export nth_cleanup=1 @@ -860,7 +860,7 @@ case ${step} in "eobs" | "eomg") export wtime_eobs="00:15:00" - export wtime_eomg="01:00:00" + export wtime_eomg="00:30:00" case ${CASE} in "C768") export npe_eobs=200;; "C384") export npe_eobs=100;; diff --git a/scripts/exgdas_enkf_fcst.sh b/scripts/exgdas_enkf_fcst.sh deleted file mode 100755 index fd6136ddd2..0000000000 --- a/scripts/exgdas_enkf_fcst.sh +++ /dev/null @@ -1,225 +0,0 @@ -#! /usr/bin/env bash - -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: exgdas_enkf_fcst.sh -# Script description: Run ensemble forecasts -# -# Author: Rahul Mahajan Org: NCEP/EMC Date: 2017-03-02 -# -# Abstract: This script runs ensemble forecasts serially one-after-another -# -# $Id$ -# -# Attributes: -# Language: POSIX shell -# -#### -################################################################################ - -source "${HOMEgfs}/ush/preamble.sh" - -# Enemble group, begin and end -ENSGRP=${ENSGRP:-1} -ENSBEG=${ENSBEG:-1} -ENSEND=${ENSEND:-1} - -# Re-run failed members, or entire group -RERUN_EFCSGRP=${RERUN_EFCSGRP:-"YES"} - -# Recenter flag and increment file prefix -RECENTER_ENKF=${RECENTER_ENKF:-"YES"} -export PREFIX_ATMINC=${PREFIX_ATMINC:-""} - -################################################################################ -# Preprocessing -cd "${DATA}" || exit 99 -DATATOP=${DATA} - -################################################################################ -# Set output data -EFCSGRP="${COM_TOP}/efcs.grp${ENSGRP}" -if [[ -f ${EFCSGRP} ]]; then - if [[ ${RERUN_EFCSGRP} = "YES" ]]; then - rm -f "${EFCSGRP}" - else - echo "RERUN_EFCSGRP = ${RERUN_EFCSGRP}, will re-run FAILED members only!" - ${NMV} "${EFCSGRP}" "${EFCSGRP}.fail" - fi -fi - -################################################################################ -# Set namelist/model config options common to all members once - -# There are many many model namelist options -# Some are resolution (CASE) dependent, some depend on the model configuration -# and will need to be added here before $FORECASTSH is called -# For now assume that -# 1. the ensemble and the deterministic are same resolution -# 2. the ensemble runs with the same configuration as the deterministic - -# Model config option for Ensemble -export TYPE=${TYPE_ENKF:-${TYPE:-nh}} # choices: nh, hydro -export MONO=${MONO_ENKF:-${MONO:-non-mono}} # choices: mono, non-mono - -# fv_core_nml -export CASE=${CASE_ENS:-${CASE:-C768}} -export layout_x=${layout_x_ENKF:-${layout_x:-8}} -export layout_y=${layout_y_ENKF:-${layout_y:-16}} -export LEVS=${LEVS_ENKF:-${LEVS:-64}} - -# nggps_diag_nml -export FHOUT=${FHOUT_ENKF:-3} -if [[ ${RUN} == "enkfgfs" ]]; then - export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT_ENKF:${FHOUT:-3}}} -fi -# model_configure -export DELTIM=${DELTIM_ENKF:-${DELTIM:-225}} -export FHMAX=${FHMAX_ENKF:-9} -if [[ ${RUN} == "enkfgfs" ]]; then - export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX_ENKF:-${FHMAX}}} -fi - -# gfs_physics_nml -export FHSWR=${FHSWR_ENKF:-${FHSWR:-3600.}} -export FHLWR=${FHLWR_ENKF:-${FHLWR:-3600.}} -export IEMS=${IEMS_ENKF:-${IEMS:-1}} -export ISOL=${ISOL_ENKF:-${ISOL:-2}} -export IAER=${IAER_ENKF:-${IAER:-111}} -export ICO2=${ICO2_ENKF:-${ICO2:-2}} -export cdmbgwd=${cdmbgwd_ENKF:-${cdmbgwd:-"3.5,0.25"}} -export dspheat=${dspheat_ENKF:-${dspheat:-".true."}} -export shal_cnv=${shal_cnv_ENKF:-${shal_cnv:-".true."}} -export FHZER=${FHZER_ENKF:-${FHZER:-6}} -export FHCYC=${FHCYC_ENKF:-${FHCYC:-6}} - -# Set PREFIX_ATMINC to r when recentering on -if [[ ${RECENTER_ENKF} = "YES" ]]; then - export PREFIX_ATMINC="r" -fi - -# Ignore possible spelling error (nothing is misspelled) -# shellcheck disable=SC2153 -GDATE=$(${NDATE} -"${assim_freq}" "${PDY}${cyc}") -declare -x gPDY="${GDATE:0:8}" -declare -x gcyc="${GDATE:8:2}" - -################################################################################ -# Run forecast for ensemble member -rc=0 -for imem in $(seq "${ENSBEG}" "${ENSEND}"); do - - cd "${DATATOP}" - - ENSMEM=$(printf %03i "${imem}") - export ENSMEM - memchar="mem${ENSMEM}" - - echo "Processing MEMBER: ${ENSMEM}" - - ra=0 - - skip_mem="NO" - if [[ -f ${EFCSGRP}.fail ]]; then - set +e - memstat=$(grep "MEMBER ${ENSMEM}" "${EFCSGRP}.fail" | grep -c "PASS") - set_strict - [[ ${memstat} -eq 1 ]] && skip_mem="YES" - fi - - # Construct COM variables from templates (see config.com) - # Can't make these read-only because we are looping over members - MEMDIR="${memchar}" YMD=${PDY} HH=${cyc} generate_com -x COM_ATMOS_RESTART COM_ATMOS_INPUT COM_ATMOS_ANALYSIS \ - COM_ATMOS_HISTORY COM_ATMOS_MASTER COM_CONF - - MEMDIR="${memchar}" YMD="${gPDY}" HH="${gcyc}" generate_com -x COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - - if [[ ${DO_WAVE} == "YES" ]]; then - MEMDIR="${memchar}" YMD=${PDY} HH=${cyc} generate_com -x COM_WAVE_RESTART COM_WAVE_PREP COM_WAVE_HISTORY - MEMDIR="${memchar}" YMD="${gPDY}" HH="${gcyc}" generate_com -x COM_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL - fi - - if [[ ${DO_OCN} == "YES" ]]; then - MEMDIR="${memchar}" YMD=${PDY} HH=${cyc} generate_com -x COM_MED_RESTART COM_OCEAN_RESTART \ - COM_OCEAN_INPUT COM_OCEAN_HISTORY COM_OCEAN_ANALYSIS - MEMDIR="${memchar}" YMD="${gPDY}" HH="${gcyc}" generate_com -x COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL - fi - - if [[ ${DO_ICE} == "YES" ]]; then - MEMDIR="${memchar}" YMD=${PDY} HH=${cyc} generate_com -x COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART - MEMDIR="${memchar}" YMD="${gPDY}" HH="${gcyc}" generate_com -x COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL - fi - - if [[ ${DO_AERO} == "YES" ]]; then - MEMDIR="${memchar}" YMD=${PDY} HH=${cyc} generate_com -x COM_CHEM_HISTORY - fi - - - if [[ ${skip_mem} = "NO" ]]; then - - ra=0 - - export MEMBER=${imem} - export DATA="${DATATOP}/${memchar}" - if [[ -d ${DATA} ]]; then rm -rf "${DATA}"; fi - mkdir -p "${DATA}" - ${FORECASTSH} - ra=$? - - # Notify a member forecast failed and abort - if [[ ${ra} -ne 0 ]]; then - err_exit "FATAL ERROR: forecast of member ${ENSMEM} FAILED. Aborting job" - fi - - rc=$((rc+ra)) - - fi - - if [[ ${SENDDBN} = YES ]]; then - fhr=${FHOUT} - while [[ ${fhr} -le ${FHMAX} ]]; do - FH3=$(printf %03i "${fhr}") - if (( fhr % 3 == 0 )); then - "${DBNROOT}/bin/dbn_alert" MODEL GFS_ENKF "${job}" "${COM_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" - fi - fhr=$((fhr+FHOUT)) - done - fi - - cd "${DATATOP}" - - if [[ -s ${EFCSGRP} ]]; then - ${NCP} "${EFCSGRP}" log_old - fi - [[ -f log ]] && rm log - [[ -f log_new ]] && rm log_new - if [[ ${ra} -ne 0 ]]; then - echo "MEMBER ${ENSMEM} : FAIL" > log - else - echo "MEMBER ${ENSMEM} : PASS" > log - fi - if [[ -s log_old ]] ; then - cat log_old log > log_new - else - cat log > log_new - fi - ${NCP} log_new "${EFCSGRP}" - -done - -################################################################################ -# Echo status of ensemble group -cd "${DATATOP}" -echo "Status of ensemble members in group ${ENSGRP}:" -cat "${EFCSGRP}" -[[ -f ${EFCSGRP}.fail ]] && rm "${EFCSGRP}".fail - -################################################################################ -# If any members failed, error out -export err=${rc}; err_chk - -################################################################################ -# Postprocessing - -exit "${err}" diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index ef1ca843f4..8f46ed6ea0 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -129,8 +129,9 @@ FV3_predet(){ LEVS=${LEVS:-65} # Other options - MEMBER=${MEMBER:-"-1"} # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER - ENS_NUM=${ENS_NUM:-1} # Single executable runs multiple members (e.g. GEFS) + # ignore errors that variable isn't used + # shellcheck disable=SC2034 + MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER PREFIX_ATMINC=${PREFIX_ATMINC:-""} # allow ensemble to use recentered increment # IAU options diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index db39c2eac5..f1beb9469d 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -618,8 +618,8 @@ if [[ ${type} == "enkfgdas" || ${type} == "enkfgfs" ]]; then fi fi done # loop over FHR - for fstep in efcs epos ; do - echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log" + for fstep in fcst epos ; do + echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log" done # eobs, ecen, esfc, and eupd are not run on the first cycle diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 154383f627..a72753eb90 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -7,7 +7,6 @@ class GEFSTasks(Tasks): def __init__(self, app_config: AppConfig, cdump: str) -> None: super().__init__(app_config, cdump) - self.nmem = self._base['NMEM_ENS'] def stage_ic(self): @@ -104,7 +103,7 @@ def fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task_name = f'fcst' + task_name = f'fcst_mem000' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -131,27 +130,29 @@ def efcs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) efcsenvars = self.envars.copy() - efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - - groups = self._get_hybgroups(self.nmem, self._configs['efcs']['NMEM_EFCSGRP']) - var_dict = {'grp': groups} + efcsenvars_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#' + } + for key, value in efcsenvars_dict.items(): + efcsenvars.append(rocoto.create_envar(name=key, value=str(value))) resources = self.get_resource('efcs') - task_name = f'efcs#grp#' + task_name = f'fcst_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': efcsenvars, 'cycledef': 'gefs', - 'command': f'{self.HOMEgfs}/jobs/rocoto/efcs.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'efmn', - 'var_dict': var_dict, + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(1, self.nmem + 1)])} + metatask_dict = {'task_name': 'fcst_ens', + 'var_dict': member_var_dict, 'task_dict': task_dict } diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index d584727431..9102c74e35 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2275,14 +2275,14 @@ def eomg(self): dependencies = rocoto.create_dependency(dep=deps) eomgenvars = self.envars.copy() - eomgenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - - groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['eobs']['NMEM_EOMGGRP']) - - var_dict = {'grp': groups} + eomgenvars_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#' + } + for key, value in eomgenvars_dict.items(): + eomgenvars.append(rocoto.create_envar(name=key, value=str(value))) resources = self.get_resource('eomg') - task_name = f'{self.cdump}eomg#grp#' + task_name = f'{self.cdump}eomg_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -2294,8 +2294,9 @@ def eomg(self): 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': f'{self.cdump}eomn', - 'var_dict': var_dict, + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(1, self.nmem + 1)])} + metatask_dict = {'task_name': f'{self.cdump}eomg', + 'var_dict': member_var_dict, 'task_dict': task_dict, } @@ -2331,7 +2332,7 @@ def eupd(self): if self.app_config.lobsdiag_forenkf: dep_dict = {'type': 'task', 'name': f'{self.cdump}ediag'} else: - dep_dict = {'type': 'metatask', 'name': f'{self.cdump}eomn'} + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}eomg'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -2539,31 +2540,30 @@ def efcs(self): dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) efcsenvars = self.envars.copy() - efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - - groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['efcs']['NMEM_EFCSGRP']) + efcsenvars_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#' + } + for key, value in efcsenvars_dict.items(): + efcsenvars.append(rocoto.create_envar(name=key, value=str(value))) - if self.cdump == "enkfgfs": - groups = self._get_hybgroups(self._base['NMEM_ENS_GFS'], self._configs['efcs']['NMEM_EFCSGRP_GFS']) cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') resources = self.get_resource('efcs') - var_dict = {'grp': groups} - - task_name = f'{self.cdump}efcs#grp#' + task_name = f'{self.cdump}fcst_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': efcsenvars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/efcs.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': f'{self.cdump}efmn', - 'var_dict': var_dict, + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(1, self.nmem + 1)])} + metatask_dict = {'task_name': f'{self.cdump}fcst', + 'var_dict': member_var_dict, 'task_dict': task_dict } @@ -2578,7 +2578,7 @@ def echgres(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}fcst'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.cdump}efcs01'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst_mem001'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2626,7 +2626,7 @@ def _get_eposgroups(epos): return grp, dep, lst deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.cdump}efmn'} + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -2675,7 +2675,9 @@ def earc(self): earcenvars = self.envars.copy() earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['earc']['NMEM_EARCGRP'], start_index=0) + # Integer division is floor division, but we need ceiling division + n_groups = -(self.nmem // -self._configs['earc']['NMEM_EARCGRP']) + groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups)]) cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index f4380fade7..110dc286b5 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -47,6 +47,7 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None: self.HOMEgfs = self._base['HOMEgfs'] self.rotdir = self._base['ROTDIR'] self.pslot = self._base['PSLOT'] + self.nmem = int(self._base['NMEM_ENS']) self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H') self.n_tiles = 6 # TODO - this needs to be elsewhere @@ -73,12 +74,6 @@ def _set_envars(envar_dict) -> list: return envars - @staticmethod - def _get_hybgroups(nens: int, nmem_per_group: int, start_index: int = 1): - ngrps = nens / nmem_per_group - groups = ' '.join([f'{x:02d}' for x in range(start_index, int(ngrps) + 1)]) - return groups - def _template_to_rocoto_cycstring(self, template: str, subs_dict: dict = {}) -> str: ''' Takes a string templated with ${ } and converts it into a string suitable diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index df9687bb72..2bc41854d8 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -316,7 +316,8 @@ def edit_baseconfig(host, inputs, yaml_dict): "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, "@gfs_cyc@": inputs.gfs_cyc, - "@APP@": inputs.app + "@APP@": inputs.app, + "@NMEM_ENS@": getattr(inputs, 'nens', 0) } tmpl_dict = dict(tmpl_dict, **extend_dict) @@ -324,7 +325,6 @@ def edit_baseconfig(host, inputs, yaml_dict): if getattr(inputs, 'nens', 0) > 0: extend_dict = { "@CASEENS@": f'C{inputs.resensatmos}', - "@NMEM_ENS@": inputs.nens, } tmpl_dict = dict(tmpl_dict, **extend_dict)