-
Notifications
You must be signed in to change notification settings - Fork 253
/
Copy pathrun_test.sh
executable file
·206 lines (163 loc) · 5.88 KB
/
run_test.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
#!/bin/bash
set -eux
echo "PID=$$"
SECONDS=0
trap '[ "$?" -eq 0 ] || write_fail_test' EXIT
trap 'echo "run_test.sh interrupted PID=$$"; cleanup' INT
trap 'echo "run_test.sh terminated PID=$$"; cleanup' TERM
cleanup() {
[[ $ROCOTO = 'false' ]] && interrupt_job
trap 0
exit
}
write_fail_test() {
if [[ ${UNIT_TEST} == true ]]; then
echo ${TEST_NR} $TEST_NAME >> $PATHRT/fail_unit_test
else
echo "${TEST_NAME} ${TEST_NR} failed in run_test" >> $PATHRT/fail_test
fi
exit 1
}
if [[ $# != 5 ]]; then
echo "Usage: $0 PATHRT RUNDIR_ROOT TEST_NAME TEST_NR COMPILE_NR"
exit 1
fi
export PATHRT=$1
export RUNDIR_ROOT=$2
export TEST_NAME=$3
export TEST_NR=$4
export COMPILE_NR=$5
cd ${PATHRT}
[[ -e ${RUNDIR_ROOT}/run_test_${TEST_NR}.env ]] && source ${RUNDIR_ROOT}/run_test_${TEST_NR}.env
source default_vars.sh
source tests/$TEST_NAME
[[ -e ${RUNDIR_ROOT}/unit_test_${TEST_NR}.env ]] && source ${RUNDIR_ROOT}/unit_test_${TEST_NR}.env
# Save original CNTL_DIR name as INPUT_DIR for regression
# tests that try to copy input data from CNTL_DIR
export INPUT_DIR=${CNTL_DIR}
# Append RT_SUFFIX to RUNDIR, and BL_SUFFIX to CNTL_DIR
export RUNDIR=${RUNDIR_ROOT}/${TEST_NAME}${RT_SUFFIX}
export CNTL_DIR=${CNTL_DIR}${BL_SUFFIX}
export JBNME=$(basename $RUNDIR_ROOT)_${TEST_NR}
echo -n "${TEST_NAME}, $( date +%s )," > ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
UNIT_TEST=${UNIT_TEST:-false}
if [[ ${UNIT_TEST} == false ]]; then
REGRESSIONTEST_LOG=${LOG_DIR}/rt_${TEST_NR}_${TEST_NAME}${RT_SUFFIX}.log
else
REGRESSIONTEST_LOG=${LOG_DIR}/ut_${TEST_NR}_${TEST_NAME}${RT_SUFFIX}.log
fi
export REGRESSIONTEST_LOG
echo "Test ${TEST_NR} ${TEST_NAME} ${TEST_DESCR}"
source rt_utils.sh
source atparse.bash
source edit_inputs.sh
mkdir -p ${RUNDIR}
cd $RUNDIR
###############################################################################
# Make configure and run files
###############################################################################
# FV3 executable:
cp ${PATHRT}/fv3_${COMPILE_NR}.exe fv3.exe
# modulefile for FV3 prerequisites:
cp ${PATHRT}/modules.fv3_${COMPILE_NR} modules.fv3
cp ${PATHTR}/modulefiles/ufs_common* .
# Get the shell file that loads the "module" command and purges modules:
cp ${PATHRT}/../NEMS/src/conf/module-setup.sh.inc module-setup.sh
SRCD="${PATHTR}"
RUND="${RUNDIR}"
# FV3_RUN could have multiple entry seperated by space
for i in ${FV3_RUN:-fv3_run.IN}
do
atparse < ${PATHRT}/fv3_conf/${i} >> fv3_run
done
if [[ $DATM_NEMS = 'true' ]] || [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then
if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then
atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml
fi
fi
atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure
atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure
if [[ "Q${INPUT_NEST02_NML:-}" != Q ]] ; then
atparse < ${PATHRT}/parm/${INPUT_NEST02_NML} > input_nest02.nml
fi
# Field Dictionary
cp ${PATHRT}/parm/fd_nems.yaml fd_nems.yaml
# Set up the run directory
source ./fv3_run
if [[ $CPLWAV == .true. ]]; then
edit_ww3_input < ${PATHRT}/parm/ww3_multi.inp.IN > ww3_multi.inp
fi
if [[ $DATM_NEMS = 'true' ]] || [[ $DATM_CDEPS = 'true' ]] || [[ $S2S = 'true' ]]; then
if [[ $HAFS = 'false' ]]; then
edit_ice_in < ${PATHRT}/parm/ice_in_template > ice_in
edit_mom_input < ${PATHRT}/parm/${MOM_INPUT:-MOM_input_template_$OCNRES} > INPUT/MOM_input
edit_diag_table < ${PATHRT}/parm/${DIAG_TABLE:-diag_table_template} > diag_table
edit_data_table < ${PATHRT}/parm/data_table_template > data_table
fi
fi
if [[ $DATM_NEMS = 'true' ]]; then
cp ${PATHRT}/parm/datm_data_table.IN datm_data_table
fi
if [[ "${DIAG_TABLE_ADDITIONAL:-}Q" != Q ]] ; then
# Append diagnostic outputs, to support tests that vary from others
# only by adding diagnostics.
atparse < "${PATHRT}/parm/${DIAG_TABLE_ADDITIONAL:-}" >> diag_table
fi
if [[ $DATM_CDEPS = 'true' ]]; then
atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in} > datm_in
atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams
fi
if [[ $DOCN_CDEPS = 'true' ]]; then
atparse < ${PATHRT}/parm/${DOCN_IN_CONFIGURE:-docn_in} > docn_in
atparse < ${PATHRT}/parm/${DOCN_STREAM_CONFIGURE:-docn.streams.IN} > docn.streams
fi
if [[ $SCHEDULER = 'pbs' ]]; then
NODES=$(( TASKS / TPN ))
if (( NODES * TPN < TASKS )); then
NODES=$(( NODES + 1 ))
fi
atparse < $PATHRT/fv3_conf/fv3_qsub.IN > job_card
elif [[ $SCHEDULER = 'slurm' ]]; then
NODES=$(( TASKS / TPN ))
if (( NODES * TPN < TASKS )); then
NODES=$(( NODES + 1 ))
fi
atparse < $PATHRT/fv3_conf/fv3_slurm.IN > job_card
elif [[ $SCHEDULER = 'lsf' ]]; then
if (( TASKS < TPN )); then
TPN=${TASKS}
fi
NODES=$(( TASKS / TPN ))
if (( NODES * TPN < TASKS )); then
NODES=$(( NODES + 1 ))
fi
atparse < $PATHRT/fv3_conf/fv3_bsub.IN > job_card
fi
################################################################################
# Submit test job
################################################################################
if [[ $SCHEDULER = 'none' ]]; then
ulimit -s unlimited
if [[ $CI_TEST = 'true' ]]; then
eval ${OMP_ENV} mpiexec -n ${TASKS} ${MPI_PROC_BIND} ./fv3.exe >out 2> >(tee err >&3)
else
mpiexec -n ${TASKS} ./fv3.exe >out 2> >(tee err >&3)
fi
else
if [[ $ROCOTO = 'false' ]]; then
submit_and_wait job_card
else
chmod u+x job_card
./job_card
fi
fi
check_results
if [[ $SCHEDULER != 'none' ]]; then
cat ${RUNDIR}/job_timestamp.txt >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
fi
################################################################################
# End test
################################################################################
echo " $( date +%s )" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
elapsed=$SECONDS
echo "Elapsed time $elapsed seconds. Test ${TEST_NAME}"