Skip to content

Commit

Permalink
Enable DA cycling on gaea C5/C6 (NOAA-EMC#3255)
Browse files Browse the repository at this point in the history 
Follow on to PR NOAA-EMC#3106 that enables DA cycling to run on gaea c5/c6
Resolves issue NOAA-EMC#2985
  • Loading branch information
@jswhit2
jswhit2 authored Jan 24, 2025
1 parent 2dd94db commit b77a4d4
Show file tree
Hide file tree
Showing 6 changed files with 415 additions and 67 deletions.
225 changes: 206 additions & 19 deletions env/GAEAC5.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,118 @@ else
exit 2
fi

if [[ "${step}" = "prep" ]]; then
case ${step} in
"prep" | "prepbufr")

export POE="NO"
export BACK="NO"
export BACK=${BACK:-"YES"}
export sys_tp="GAEAC5"
export launcher_PREP="srun"
;;
"prep_emissions")

elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
export APRUN="${APRUN_default}"
;;
"waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")

export CFP_MP="YES"
[[ "${step}" = "waveprep" ]] && export MP_PULSE=0
export wavempexec=${launcher}
export wave_mpmd=${mpmd_opt}

;;
"atmanlvar")

export NTHREADS_ATMANLVAR=${NTHREADSmax}
export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}"
;;
"atmanlfv3inc")

export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
;;
"atmensanlobs")

export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
export APRUN_ATMENSANLOBS="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
;;
"atmensanlsol")

export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
export APRUN_ATMENSANLSOL="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
;;
"atmensanlletkf")

export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}"
;;
"atmensanlfv3inc")

export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
;;
"aeroanlvar")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_AEROANL=${NTHREADSmax}
export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}"
;;
"aeroanlgenb")

export NTHREADS_AEROANLGENB=${NTHREADSmax}
export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}"
;;
"prepobsaero")

export NTHREADS_PREPOBSAERO=${NTHREADS1}
export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
;;
"snowanl")

export APRUN_CALCFIMS="${launcher} -n 1"

export NTHREADS_SNOWANL=${NTHREADSmax}
export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}"

export APRUN_APPLY_INCR="${launcher} -n 6"
;;
"esnowrecen")

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"

export APRUN_APPLY_INCR="${launcher} -n 6"
;;

"marinebmat")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
export APRUN_MARINEBMAT="${APRUN_default}"
;;
"marineanlvar")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
export APRUN_MARINEANLVAR="${APRUN_default}"
;;
"ocnanalecen")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

max_threads_per_task=$((max_tasks_per_node / tasks_per_node_ocnanalecen))

export NTHREADS_OCNANALECEN=${threads_per_task_ocnanalecen:-${max_threads_per_task}}
[[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task}
export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
;;
"marineanlchkpt")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_OCNANAL=${NTHREADSmax}
export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
;;
"anal" | "analcalc")

export MKL_NUM_THREADS=4
export MKL_CBWR=AUTO
Expand All @@ -50,7 +154,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_GSI=${NTHREADSmax}
export NTHREADS_GSI=${threads_per_task_anal:-${max_threads_per_task}}
export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"

export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
Expand All @@ -65,44 +169,127 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
export NTHREADS_GAUSFCANL=1
ntasks_gausfcanl=${ntasks_gausfcanl:-1}
export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"

elif [[ "${step}" = "sfcanl" ]]; then
;;
"sfcanl")

export NTHREADS_CYCLE=${threads_per_task:-14}
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
;;
"eobs")

export MKL_NUM_THREADS=4
export MKL_CBWR=AUTO

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"


export NTHREADS_GSI=${NTHREADSmax}
[[ ${NTHREADS_GSI} -gt ${max_threads_per_task} ]] && export NTHREADS_GSI=${max_threads_per_task}
export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"
;;
"eupd")

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"


export NTHREADS_ENKF=${NTHREADSmax}
export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}"
;;
"fcst" | "efcs")

elif [[ "${step}" = "fcst" ]]; then
export OMP_STACKSIZE=512M

(( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
(( ufs_ntasks = nnodes*tasks_per_node ))
# With ESMF threading, the model wants to use the full node
export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
unset nnodes ufs_ntasks
;;

elif [[ "${step}" = "upp" ]]; then
"upp")

export NTHREADS_UPP=${NTHREADS1}
export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}"
;;

elif [[ "${step}" = "atmos_products" ]]; then
"atmos_products")

export USE_CFP="YES" # Use MPMD for downstream product generation on GaeaC5
export USE_CFP="YES" # Use MPMD for downstream product generation
;;

elif [[ "${step}" = "oceanice_products" ]]; then
"oceanice_products")

export NTHREADS_OCNICEPOST=${NTHREADS1}
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
;;

elif [[ "${step}" = "fit2obs" ]]; then
"ecen")

export NTHREADS_FIT2OBS=${NTHREADS1}
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
export NTHREADS_ECEN=${NTHREADSmax}
export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}"

export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
[[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
export APRUN_CHGRES="time"

export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
[[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}"

;;
"esfc")

elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
export NTHREADS_ESFC=${NTHREADSmax}
export APRUN_ESFC="${APRUN_default} --cpus-per-task=${NTHREADS_ESFC}"

export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"

;;
"epos")

export NTHREADS_EPOS=${NTHREADSmax}
export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}"

;;
"postsnd")

export CFP_MP="YES"
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
export wavempexec=${launcher}
export wave_mpmd=${mpmd_opt}

fi
export NTHREADS_POSTSND=${NTHREADS1}
export APRUN_POSTSND="${APRUN_default} --cpus-per-task=${NTHREADS_POSTSND}"

export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
[[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}
export APRUN_POSTSNDCFP="${launcher} -n ${ntasks_postsndcfp} ${mpmd_opt}"

;;
"awips")

export NTHREADS_AWIPS=${NTHREADS1}
export APRUN_AWIPSCFP="${APRUN_default} ${mpmd_opt}"

;;
"gempak")

echo "WARNING: ${step} is not enabled on ${machine}!"

;;
"fit2obs")

export NTHREADS_FIT2OBS=${NTHREADS1}
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"

;;
*)
# Some other job not yet defined here
echo "WARNING: The job step ${step} does not specify GAEAC5-specific resources"
;;
esac

Loading

0 comments on commit b77a4d4

Please sign in to comment.