Use stochastic restart patterns on rerun

The stochastic pattern restart files were not being copied into the
input directory when restarting the model after a segment/failure.
These files are now copied in. Additional changes appear to be
needed for them to be used by UFS.

The files are NOT copied in for non-rerun warm starts.

Refs #2937

parm/config/gefs/config.fcst

@@ -237,11 +237,13 @@ case ${imp_physics} in
 esac
 
 # Stochastic physics
-export DO_SPPT=${DO_SPPT:-"NO"}
-export DO_SKEB=${DO_SKEB:-"NO"}
-export DO_SHUM=${DO_SHUM:-"NO"}
-export DO_LAND_PERT=${DO_LAND_PERT:-"NO"}
-export DO_CA=${DO_CA:-"YES"}
+export DO_SPPT="NO"
+export DO_SKEB="NO"
+export DO_SHUM="NO"
+export DO_LAND_PERT="NO"
+export DO_CA="NO"
+export DO_OCN_SPPT="NO"
+export DO_OCN_PERT_EPBL="NO"
 
 #coupling settings
 export cplmode="ufs.frac"

parm/config/gfs/config.efcs

@@ -59,6 +59,7 @@ export SPPT_TAU=21600.
 export SPPT_LSCALE=500000.
 export SPPT_LOGIT=".true."
 export SPPT_SFCLIMIT=".true."
+export DO_CA="YES"
 
 if [[ "${QUILTING}" == ".true." ]] && [[ "${OUTPUT_GRID}" == "gaussian_grid" ]]; then
     export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_da"

parm/config/gfs/config.fcst

@@ -257,14 +257,13 @@ case ${imp_physics} in
 esac
 
 # Stochastic physics
-export DO_SPPT=${DO_SPPT:-"NO"}
-export DO_SKEB=${DO_SKEB:-"NO"}
-export DO_SHUM=${DO_SHUM:-"NO"}
-export DO_LAND_PERT=${DO_LAND_PERT:-"NO"}
-export DO_CA=${DO_CA:-"YES"}
-if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
-  export DO_CA="NO"  # CA does not work with nesting.
-fi
+export DO_SPPT="NO"
+export DO_SKEB="NO"
+export DO_SHUM="NO"
+export DO_LAND_PERT="NO"
+export DO_CA="NO"
+export DO_OCN_SPPT="NO"
+export DO_OCN_PERT_EPBL="NO"
 
 #coupling settings
 export cplmode="ufs.frac"

ush/forecast_postdet.sh

@@ -47,7 +47,15 @@ FV3_postdet() {
       || ( echo "FATAL ERROR: Unable to copy FV3 IC, ABORT!"; exit 1 )
     done
 
-    if [[ "${RERUN}" != "YES" ]]; then
+    if [[ "${RERUN}" == "YES" ]]; then
+      file_list=$(stoch_restarts)
+      echo "Copying stochastic restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'"
+      for stoch_file in $(stoch_restarts); do
+        restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${stoch_file}"
+        ${NCP} "${restart_dir}/${restart_file}" "${DATA}/INPUT/${stoch_file}" \
+        || ( echo "FATAL ERROR: Unable to copy stochastic restart, ABORT!"; exit 1 )
+      done
+    else
       # Replace sfc_data with sfcanl_data restart files from current cycle (if found)
       local nn
       for (( nn = 1; nn <= ntiles; nn++ )); do
@@ -78,7 +86,7 @@ FV3_postdet() {
                  "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc"
         done
       fi # if [[ ${use_anl_aero} == "YES" ]]; then
-    fi  # if [[ "${RERUN}" != "YES" ]]; then
+    fi  # if [[ "${RERUN}" == "YES" ]]; then
 
   fi  # if [[ "${warm_start}" == ".true." ]]; then
 

ush/forecast_predet.sh

@@ -68,6 +68,22 @@ FV3_restarts(){
   IFS=, echo "${fv3_restart_files[*]}"
 }
 
+stoch_restarts(){
+  # These only get copied for reruns
+  local stoch_restart_files
+  stoch_restart_files=(  )
+
+  if [[ "${DO_SPPT:-}" == "YES" || "${DO_SKEB:-}" == "YES" || \
+        "${DO_SHUM:-}" == "YES" || "${DO_LAND_PERT:-}" == "YES" ]]; then
+    stoch_restart_files+=(atm_stoch.res.nc)
+  fi
+  if [[ "${DO_OCN:-}" == "YES" && ( "${DO_OCN_SPPT:-}" == "YES" || "${DO_OCN_PERT_EPBL}" == "YES" ) ]]; then
+    stoch_restart_files+=(ocn_stoch.res.nc)
+  fi
+  # Create a comma separated string from array using IFS
+  IFS=, echo "${stoch_restart_files[*]}"
+}
+
 # shellcheck disable=SC2034
 common_predet(){
   echo "SUB ${FUNCNAME[0]}: Defining variables for shared through model components"
@@ -377,41 +393,38 @@ FV3_predet(){
   do_sppt=".false."
   do_ca=".false."
   ISEED=0
-  if (( MEMBER > 0 )); then  # these are only applicable for ensemble members
-    local imem=${MEMBER#0}
-    local base_seed=$((current_cycle*10000 + imem*100))
+  local imem=${MEMBER#0}
+  local base_seed=$((current_cycle*10000 + imem*100))
 
-    if [[ "${DO_SKEB:-}" == "YES" ]]; then
-      do_skeb=".true."
-      ISEED_SKEB=$((base_seed + 1))
-    fi
-
-    if [[ "${DO_SHUM:-}" == "YES" ]]; then
-      do_shum=".true."
-      ISEED_SHUM=$((base_seed + 2))
-    fi
+  if [[ "${DO_SKEB:-}" == "YES" ]]; then
+    do_skeb=".true."
+    ISEED_SKEB=$((base_seed + 1))
+  fi
 
-    if [[ "${DO_SPPT:-}" == "YES" ]]; then
-      do_sppt=".true."
-      ISEED_SPPT=$((base_seed + 3)),$((base_seed + 4)),$((base_seed + 5)),$((base_seed + 6)),$((base_seed + 7))
-    fi
+  if [[ "${DO_SHUM:-}" == "YES" ]]; then
+    do_shum=".true."
+    ISEED_SHUM=$((base_seed + 2))
+  fi
 
-    if [[ "${DO_CA:-}" == "YES" ]]; then
-      do_ca=".true."
-      ISEED_CA=$(( (base_seed + 18) % 2147483647 ))
-    fi
+  if [[ "${DO_SPPT:-}" == "YES" ]]; then
+    do_sppt=".true."
+    ISEED_SPPT=$((base_seed + 3)),$((base_seed + 4)),$((base_seed + 5)),$((base_seed + 6)),$((base_seed + 7))
+  fi
 
-    if [[ "${DO_LAND_PERT:-}" == "YES" ]]; then
-      lndp_type=${lndp_type:-2}
-      ISEED_LNDP=$(( (base_seed + 5) % 2147483647 ))
-      LNDP_TAU=${LNDP_TAU:-21600}
-      LNDP_SCALE=${LNDP_SCALE:-500000}
-      lndp_var_list=${lndp_var_list:-"'smc', 'vgf',"}
-      lndp_prt_list=${lndp_prt_list:-"0.2,0.1"}
-      n_var_lndp=$(echo "${lndp_var_list}" | wc -w)
-    fi
+  if [[ "${DO_CA:-}" == "YES" ]]; then
+    do_ca=".true."
+    ISEED_CA=$(( (base_seed + 18) % 2147483647 ))
+  fi
 
-  fi  # end of ensemble member specific options
+  if [[ "${DO_LAND_PERT:-}" == "YES" ]]; then
+    lndp_type=${lndp_type:-2}
+    ISEED_LNDP=$(( (base_seed + 5) % 2147483647 ))
+    LNDP_TAU=${LNDP_TAU:-21600}
+    LNDP_SCALE=${LNDP_SCALE:-500000}
+    lndp_var_list=${lndp_var_list:-"'smc', 'vgf',"}
+    lndp_prt_list=${lndp_prt_list:-"0.2,0.1"}
+    n_var_lndp=$(echo "${lndp_var_list}" | wc -w)
+  fi
 
   #--------------------------------------------------------------------------
 
@@ -680,7 +693,7 @@ MOM6_predet(){
 
   # If using stochastic parameterizations, create a seed that does not exceed the
   # largest signed integer
-  if (( MEMBER > 0 )); then  # these are only applicable for ensemble members
+  # if (( MEMBER > 0 )); then  # these are only applicable for ensemble members
     local imem=${MEMBER#0}
     local base_seed=$((current_cycle*10000 + imem*100))
 
@@ -691,7 +704,7 @@ MOM6_predet(){
     if [[ "${DO_OCN_PERT_EPBL:-}" == "YES" ]]; then
       ISEED_EPBL=$((base_seed + 13)),$((base_seed + 14)),$((base_seed + 15)),$((base_seed + 16)),$((base_seed + 17))
     fi
-  fi
+  # fi
 
   # Fix files
   ${NCP} "${FIXgfs}/mom6/${OCNRES}/"* "${DATA}/INPUT/"  # TODO: These need to be explicit