Skip to content

Commit 5ac6836

Browse files
DavidHuber-NOAADavidHuber-NOAA
authored
Update S4 environment and module files (#1303)
Updates the S4 module file and environment file to keep up to date with the current develop branch. Fixes #1297. Addresses one bug in #1195. This also increases the memory request for the ediag job. During testing, that job initially failed. For the 5 cycles run between 2022051500 - 2022051600, the largest memory footprint was ~26.5GB.
1 parent 1040216 commit 5ac6836

File tree

4 files changed

+78
-85
lines changed

4 files changed

+78
-85
lines changed

‎env/S4.env‎

Lines changed: 74 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
#!/usr/bin/env bash
1+
#! /usr/bin/env bash
22

33
if [[ $# -ne 1 ]]; then
44

55
echo "Must specify an input argument to set runtime environment variables!"
66
echo "argument can be any one of the following:"
77
echo "atmanalrun atmensanalrun"
8+
echo "aeroanlrun"
89
echo "anal sfcanl fcst post vrfy metp"
910
echo "eobs eupd ecen efcs epos"
1011
echo "postsnd awips gempak"
@@ -30,264 +31,256 @@ export NTHSTACK=1024000000
3031
ulimit -s unlimited
3132
ulimit -a
3233

33-
if [[ ${step} = "prep" || ${step} = "prepbufr" ]]; then
34+
if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
3435

35-
npe_node_prep=${npe_node_prep:-${npe_node_max}}
3636
nth_max=$((npe_node_max / npe_node_prep))
3737

3838
export POE="NO"
3939
export BACK="NO"
4040
export sys_tp="S4"
4141
export launcher_PREP="srun"
4242

43-
elif [[ ${step} = "waveinit" || ${step} = "waveprep" || ${step} = "wavepostsbs" || ${step} = "wavepostbndpnt" || ${step} = "wavepostbndpntbll" || ${step} = "wavepostpnt" ]]; then
43+
elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
4444

4545
export CFP_MP="YES"
46-
if [[ ${step} = "waveprep" ]]; then export MP_PULSE=0 ; fi
46+
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
4747
export wavempexec=${launcher}
4848
export wave_mpmd=${mpmd_opt}
4949

50-
elif [[ ${step} = "atmanalrun" ]]; then
50+
elif [[ "${step}" = "atmanalrun" ]]; then
5151

5252
export CFP_MP=${CFP_MP:-"YES"}
5353
export USE_CFP=${USE_CFP:-"YES"}
54-
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
54+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
5555

56-
npe_node_atmanalrun=${npe_node_atmanalrun:-${npe_node_max}}
5756
nth_max=$((npe_node_max / npe_node_atmanalrun))
5857

5958
export NTHREADS_ATMANAL=${nth_atmanalrun:-${nth_max}}
6059
[[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max}
61-
export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun:-0}"
60+
export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}"
6261

63-
elif [[ ${step} = "atmensanalrun" ]]; then
62+
elif [[ "${step}" = "atmensanalrun" ]]; then
6463

6564
export CFP_MP=${CFP_MP:-"YES"}
6665
export USE_CFP=${USE_CFP:-"YES"}
67-
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
66+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
6867

69-
npe_node_atmensanalrun=${npe_node_atmensanalrun:-${npe_node_max}}
7068
nth_max=$((npe_node_max / npe_node_atmensanalrun))
7169

7270
export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}}
7371
[[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max}
74-
export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun:-0}"
72+
export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}"
7573

76-
elif [[ ${step} = "aeroanlrun" ]]; then
74+
elif [[ "${step}" = "aeroanlrun" ]]; then
7775

78-
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
76+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
7977

80-
npe_node_aeroanlrun=${npe_node_aeroanlrun:-${npe_node_max}}
8178
nth_max=$((npe_node_max / npe_node_aeroanlrun))
8279

8380
export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
8481
[[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
85-
export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun:-0}"
82+
export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
8683

87-
elif [[ ${step} = "anal" || ${step} = "analcalc" ]]; then
84+
elif [[ "${step}" = "ocnanalbmat" ]]; then
85+
echo "WARNING: ${step} is not enabled on S4!"
86+
87+
elif [[ "${step}" = "ocnanalrun" ]]; then
88+
echo "WARNING: ${step} is not enabled on S4!"
89+
90+
elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
8891

8992
export MKL_NUM_THREADS=4
9093
export MKL_CBWR=AUTO
9194

9295
export CFP_MP=${CFP_MP:-"YES"}
9396
export USE_CFP=${USE_CFP:-"YES"}
94-
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
97+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
9598

96-
npe_node_anal=${npe_node_anal:-${npe_node_max}}
9799
nth_max=$((npe_node_max / npe_node_anal))
98100

99101
export NTHREADS_GSI=${nth_anal:-${nth_max}}
100102
[[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
101-
export APRUN_GSI=${launcher}
103+
export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}}"
102104

103105
export NTHREADS_CALCINC=${nth_calcinc:-1}
104106
[[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
105-
export APRUN_CALCINC=${launcher}
107+
export APRUN_CALCINC="${launcher} \$ncmd"
106108

107109
export NTHREADS_CYCLE=${nth_cycle:-12}
108110
[[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
109111
npe_cycle=${ntiles:-6}
110-
export APRUN_CYCLE="${launcher} -n ${npe_cycle:-0}"
112+
export APRUN_CYCLE="${launcher} -n ${npe_cycle}"
111113

112114

113115
export NTHREADS_GAUSFCANL=1
114116
npe_gausfcanl=${npe_gausfcanl:-1}
115-
export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl:-0}"
117+
export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
116118

117-
elif [[ ${step} = "sfcanl" ]]; then
118-
npe_node_sfcanl=${npe_node_sfcanl:-${npe_node_max}}
119+
elif [[ "${step}" = "sfcanl" ]]; then
119120
nth_max=$((npe_node_max / npe_node_sfcanl))
120121

121122
export NTHREADS_CYCLE=${nth_sfcanl:-14}
122123
[[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
123124
npe_sfcanl=${ntiles:-6}
124-
export APRUN_CYCLE="${launcher} -n ${npe_sfcanl:-0}"
125+
export APRUN_CYCLE="${launcher} -n ${npe_sfcanl}"
125126

126-
elif [[ ${step} = "gldas" ]]; then
127+
elif [[ "${step}" = "gldas" ]]; then
128+
129+
export USE_CFP="NO"
130+
export CFP_MP="YES"
127131

128-
npe_node_gldas=${npe_node_gldas:-${npe_node_max}}
129132
nth_max=$((npe_node_max / npe_node_gldas))
130133

131134
export NTHREADS_GLDAS=${nth_gldas:-${nth_max}}
132135
[[ ${NTHREADS_GLDAS} -gt ${nth_max} ]] && export NTHREADS_GLDAS=${nth_max}
133-
export APRUN_GLDAS="${launcher} -n ${npe_gldas:-0}"
136+
export APRUN_GLDAS="${launcher} -n ${npe_gldas}"
134137

135138
export NTHREADS_GAUSSIAN=${nth_gaussian:-1}
136139
[[ ${NTHREADS_GAUSSIAN} -gt ${nth_max} ]] && export NTHREADS_GAUSSIAN=${nth_max}
137-
export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian:-0}"
140+
export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian}"
138141

139142
# Must run data processing with exactly the number of tasks as time
140143
# periods being processed.
141144

142-
gldas_spinup_hours=${gldas_spinup_hours:-0}
143145
npe_gldas_data_proc=$((gldas_spinup_hours + 12))
144-
export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} --multi-prog"
146+
export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} ${mpmd_opt}"
145147

146-
elif [[ ${step} = "eobs" ]]; then
148+
elif [[ "${step}" = "eobs" ]]; then
147149

148150
export MKL_NUM_THREADS=4
149151
export MKL_CBWR=AUTO
150152

151-
export CFP_MP=${CFP_MP:-"YES"}
152-
export USE_CFP=${USE_CFP:-"YES"}
153-
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
154-
155-
npe_node_eobs=${npe_node_eobs:-${npe_node_max}}
156153
nth_max=$((npe_node_max / npe_node_eobs))
157154

158155
export NTHREADS_GSI=${nth_eobs:-${nth_max}}
159156
[[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
160-
export APRUN_GSI=${launcher}
161-
162-
elif [[ ${step} = "eupd" ]]; then
157+
export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_eobs}}"
163158

164159
export CFP_MP=${CFP_MP:-"YES"}
165160
export USE_CFP=${USE_CFP:-"YES"}
166-
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
161+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
162+
163+
elif [[ "${step}" = "eupd" ]]; then
167164

168-
npe_node_eupd=${npe_node_eupd:-${npe_node_max}}
169165
nth_max=$((npe_node_max / npe_node_eupd))
170166

171167
export NTHREADS_ENKF=${nth_eupd:-${nth_max}}
172168
[[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max}
173-
export APRUN_ENKF=${launcher}
169+
export APRUN_ENKF="${launcher} -n ${npe_enkf:-${npe_eupd}}"
170+
171+
export CFP_MP=${CFP_MP:-"YES"}
172+
export USE_CFP=${USE_CFP:-"YES"}
173+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
174174

175-
elif [[ ${step} = "fcst" ]]; then
175+
elif [[ "${step}" = "fcst" ]]; then
176176

177177
#PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
178-
if [[ ${CDUMP:-gdas} == "gfs" ]]; then
179-
npe_fcst=${npe_fcst_gfs:-0}
180-
npe_node_fcst=${npe_node_fcst_gfs:-${npe_node_max}}
181-
nth_fv3=${nth_fv3_gfs:-1}
178+
if [[ ${CDUMP} == "gfs" ]]; then
179+
npe_fcst=${npe_fcst_gfs}
180+
npe_node_fcst=${npe_node_fcst_gfs}
181+
nth_fv3=${nth_fv3_gfs}
182182
fi
183183

184-
npe_node_fcst=${npe_node_fcst:-${npe_node_max}}
185184
nth_max=$((npe_node_max / npe_node_fcst))
186185

187186
export NTHREADS_FV3=${nth_fv3:-${nth_max}}
188187
[[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max}
189188
export cores_per_node=${npe_node_max}
190-
export APRUN_FV3="${launcher} -n ${npe_fcst:-0}"
189+
export APRUN_FV3="${launcher} -n ${npe_fcst}"
191190

192191
export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
193192
[[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max}
194-
export APRUN_REGRID_NEMSIO=${launcher}
193+
export APRUN_REGRID_NEMSIO="${launcher} -n ${LEVS}"
195194

196195
export NTHREADS_REMAP=${nth_remap:-2}
197196
[[ ${NTHREADS_REMAP} -gt ${nth_max} ]] && export NTHREADS_REMAP=${nth_max}
198-
export APRUN_REMAP=${launcher}
197+
export APRUN_REMAP="${launcher} -n ${npe_remap:-${npe_fcst}}"
199198
export I_MPI_DAPL_UD="enable"
200199

201-
elif [[ ${step} = "efcs" ]]; then
200+
elif [[ "${step}" = "efcs" ]]; then
202201

203-
npe_node_efcs=${npe_node_efcs:-${npe_node_max}}
204202
nth_max=$((npe_node_max / npe_node_efcs))
205203

206204
export NTHREADS_FV3=${nth_efcs:-${nth_max}}
207205
[[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max}
208206
export cores_per_node=${npe_node_max}
209-
export APRUN_FV3="${launcher} -n ${npe_efcs:-0}"
207+
export APRUN_FV3="${launcher} -n ${npe_efcs}"
210208

211209
export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
212210
[[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max}
213-
export APRUN_REGRID_NEMSIO="${launcher} ${LEVS:-128}"
211+
export APRUN_REGRID_NEMSIO="${launcher} -n ${LEVS}"
214212

215-
elif [[ ${step} = "post" ]]; then
213+
elif [[ "${step}" = "post" ]]; then
216214

217-
npe_node_post=${npe_node_post:-npe_node_max}
218215
nth_max=$((npe_node_max / npe_node_post))
219216

220217
export NTHREADS_NP=${nth_np:-1}
221218
[[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
222-
export APRUN_NP=${launcher}
219+
export APRUN_NP="${launcher} -n ${npe_post}"
223220

224221
export NTHREADS_DWN=${nth_dwn:-1}
225222
[[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
226-
export APRUN_DWN=${launcher}
223+
export APRUN_DWN="${launcher} -n ${npe_dwn}"
227224

228-
elif [[ ${step} = "ecen" ]]; then
225+
elif [[ "${step}" = "ecen" ]]; then
229226

230-
npe_node_ecen=${npe_node_ecen:-${npe_node_max}}
231227
nth_max=$((npe_node_max / npe_node_ecen))
232228

233229
export NTHREADS_ECEN=${nth_ecen:-${nth_max}}
234230
[[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max}
235-
export APRUN_ECEN=${launcher}
231+
export APRUN_ECEN="${launcher} -n ${npe_ecen}"
236232

237233
export NTHREADS_CHGRES=${nth_chgres:-12}
238234
[[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max}
239235
export APRUN_CHGRES="time"
240236

241237
export NTHREADS_CALCINC=${nth_calcinc:-1}
242238
[[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
243-
export APRUN_CALCINC=${launcher}
239+
export APRUN_CALCINC="${launcher} -n ${npe_ecen}"
244240

245-
elif [[ ${step} = "esfc" ]]; then
241+
elif [[ "${step}" = "esfc" ]]; then
246242

247-
npe_node_esfc=${npe_node_esfc:-${npe_node_max}}
248243
nth_max=$((npe_node_max / npe_node_esfc))
249244

250245
export NTHREADS_ESFC=${nth_esfc:-${nth_max}}
251246
[[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max}
252-
export APRUN_ESFC="${launcher} -n ${npe_esfc:-0}"
247+
export APRUN_ESFC="${launcher} -n ${npe_esfc}"
253248

254249
export NTHREADS_CYCLE=${nth_cycle:-14}
255250
[[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
256-
export APRUN_CYCLE="${launcher} -n ${npe_esfc:-0}"
251+
export APRUN_CYCLE="${launcher} -n ${npe_esfc}"
257252

258-
elif [[ ${step} = "epos" ]]; then
253+
elif [[ "${step}" = "epos" ]]; then
259254

260-
npe_node_epos=${npe_node_epos:-${npe_node_max}}
261255
nth_max=$((npe_node_max / npe_node_epos))
262256

263257
export NTHREADS_EPOS=${nth_epos:-${nth_max}}
264258
[[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max}
265-
export APRUN_EPOS=${launcher}
259+
export APRUN_EPOS="${launcher} -n ${npe_epos}"
266260

267-
elif [[ ${step} = "init" ]]; then
261+
elif [[ "${step}" = "init" ]]; then
268262

269-
export APRUN=${launcher}
263+
export APRUN="${launcher} -n ${npe_init}"
270264

271-
elif [[ ${step} = "postsnd" ]]; then
265+
elif [[ "${step}" = "postsnd" ]]; then
272266

273267
export CFP_MP="YES"
274268

275-
npe_node_postsnd=${npe_node_postsnd:-${npe_node_max}}
276269
nth_max=$((npe_node_max / npe_node_postsnd))
277270

278271
export NTHREADS_POSTSND=${nth_postsnd:-1}
279272
[[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max}
280-
export APRUN_POSTSND=${launcher}
273+
export APRUN_POSTSND="${launcher} -n ${npe_postsnd}"
281274

282275
export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
283276
[[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max}
284-
export APRUN_POSTSNDCFP=${launcher}
277+
export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}"
285278

286-
elif [[ ${step} = "awips" ]]; then
279+
elif [[ "${step}" = "awips" ]]; then
287280

288281
echo "WARNING: ${step} is not enabled on S4!"
289282

290-
elif [[ ${step} = "gempak" ]]; then
283+
elif [[ "${step}" = "gempak" ]]; then
291284

292285
echo "WARNING: ${step} is not enabled on S4!"
293286
fi

‎modulefiles/module_base.s4.lua‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ load(pathJoin("png", "1.6.35"))
2424

2525
load(pathJoin("hdf5", "1.10.6"))
2626
load(pathJoin("netcdf", "4.7.4"))
27-
load(pathJoin("pio", "2.5.2"))
28-
load(pathJoin("esmf", "8.2.1b04"))
27+
load(pathJoin("pio", "2.5.7"))
28+
load(pathJoin("esmf", "8.3.0b09"))
2929
load(pathJoin("fms", "2021.03"))
3030

3131
load(pathJoin("bacio", "2.4.1"))

‎parm/config/config.resources‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -729,7 +729,7 @@ elif [ ${step} = "ediag" ]; then
729729
export npe_ediag=48
730730
export nth_ediag=1
731731
export npe_node_ediag=$(echo "${npe_node_max} / ${nth_ediag}" | bc)
732-
export memory_ediag="24GB"
732+
export memory_ediag="30GB"
733733

734734
elif [ ${step} = "eupd" ]; then
735735

‎scripts/exgdas_enkf_earc.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ if (( 10#${ENSGRP} > 0 )) && [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]];
4848
SAVEWARMICB="NO"
4949
mm=$(echo "${CDATE}"|cut -c 5-6)
5050
dd=$(echo "${CDATE}"|cut -c 7-8)
51-
nday=$(( (mm-1)*30+dd ))
51+
nday=$(( (10#${mm}-1)*30+10#${dd} ))
5252
mod=$((nday % ARCH_WARMICFREQ))
5353
if [ "${CDATE}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCINC_CYC}" ]; then SAVEWARMICA="YES" ; fi
5454
if [ "${CDATE}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCICS_CYC}" ]; then SAVEWARMICB="YES" ; fi

0 commit comments

Comments
 (0)