Skip to content

Commit 7ca45db

Browse files
HenryRWinterbottomHenryRWinterbottom
authored
Merge branch 'NOAA-EMC:develop' into feature/gfsv17_issue_2125
2 parents 73bc76b + a23b7f2 commit 7ca45db

37 files changed

+229
-88
lines changed

‎ci/cases/pr/C48C48_ufs_hybatmDA.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
experiment:
2+
system: gfs
3+
mode: cycled
4+
5+
arguments:
6+
pslot: {{ 'pslot' | getenv }}
7+
app: ATM
8+
resdetatmos: 48
9+
resensatmos: 48
10+
comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
11+
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
12+
idate: 2021032312
13+
edate: 2021032400
14+
nens: 2
15+
gfs_cyc: 1
16+
start: warm
17+
yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
18+
19+
skip_ci_on_hosts:
20+
- hera
21+
- orion
22+
- hercules

‎ci/cases/pr/C48_S2SWA_gefs.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,8 @@ arguments:
1616
idate: 2021032312
1717
edate: 2021032312
1818
yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
19+
20+
skip_ci_on_hosts:
21+
- hera
22+
- orion
23+
- hercules

‎ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ arguments:
1717
start: warm
1818
yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
1919

20-
skip_ci_on_host:
20+
skip_ci_on_hosts:
2121
- orion
2222
- hera
2323
- hercules

‎ci/cases/yamls/build.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
builds:
22
- gefs: './build_all.sh'
3-
- gfs: './build_all.sh -gu'
3+
- gfs: './build_all.sh -wgu'

‎ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
defaults:
2+
!INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
3+
base:
4+
DOIAU: "NO"
5+
DO_JEDIATMVAR: "YES"
6+
DO_JEDIATMENS: "YES"
7+
ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
8+
esfc:
9+
DONST: "NO"
10+
nsst:
11+
NST_MODEL: "1"
12+
sfcanl:
13+
DONST: "NO"
14+

‎ci/scripts/check_ci.sh

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ set -eux
88
# to run from within a cron job in the CI Managers account
99
#####################################################################################
1010

11-
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
11+
HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
1212
scriptname=$(basename "${BASH_SOURCE[0]}")
1313
echo "Begin ${scriptname} at $(date -u)" || true
1414
export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -20,21 +20,22 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
2020
# Set up runtime environment varibles for accounts on supproted machines
2121
#########################################################################
2222

23-
source "${ROOT_DIR}/ush/detect_machine.sh"
23+
source "${HOMEgfs}/ush/detect_machine.sh"
2424
case ${MACHINE_ID} in
2525
hera | orion | hercules)
2626
echo "Running Automated Testing on ${MACHINE_ID}"
27-
source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
27+
source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
2828
;;
2929
*)
3030
echo "Unsupported platform. Exiting with error."
3131
exit 1
3232
;;
3333
esac
3434
set +x
35-
source "${ROOT_DIR}/ush/module-setup.sh"
36-
source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
37-
module use "${ROOT_DIR}/modulefiles"
35+
export HOMEgfs
36+
source "${HOMEgfs}/ush/module-setup.sh"
37+
source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
38+
module use "${HOMEgfs}/modulefiles"
3839
module load "module_gwsetup.${MACHINE_ID}"
3940
module list
4041
set -x
@@ -57,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
5758

5859
pr_list=""
5960
if [[ -f "${pr_list_dbfile}" ]]; then
60-
pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
61+
pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
6162
fi
6263
if [[ -z "${pr_list+x}" ]]; then
6364
echo "no PRs open and ready to run cases on .. exiting"
@@ -89,13 +90,13 @@ for pr in ${pr_list}; do
8990
sed -i "1 i\`\`\`" "${output_ci}"
9091
sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
9192
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
92-
"${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
93+
"${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
9394
# Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
9495
weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}" --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
9596
if [[ -n "${weekly_labels}" ]]; then
96-
num_platforms=$(find "${ROOT_DIR}/ci/platforms" -type f -name "config.*" | wc -l)
97+
num_platforms=$(find "${HOMEgfs}/ci/platforms" -type f -name "config.*" | wc -l)
9798
passed=0
98-
for platforms in "${ROOT_DIR}"/ci/platforms/config.*; do
99+
for platforms in "${HOMEgfs}"/ci/platforms/config.*; do
99100
machine=$(basename "${platforms}" | cut -d. -f2)
100101
if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then
101102
((passed=passed+1))
@@ -139,7 +140,7 @@ for pr in ${pr_list}; do
139140
} >> "${output_ci}"
140141
sed -i "1 i\`\`\`" "${output_ci}"
141142
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
142-
"${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
143+
"${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
143144
for kill_cases in "${pr_dir}/RUNTESTS/"*; do
144145
pslot=$(basename "${kill_cases}")
145146
cancel_slurm_jobs "${pslot}"

‎ci/scripts/clone-build_ci.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ set +e
7474
source "${HOMEgfs}/ush/module-setup.sh"
7575
export BUILD_JOBS=8
7676
rm -rf log.build
77-
./build_all.sh -gu >> log.build 2>&1
77+
./build_all.sh -guw >> log.build 2>&1
7878
build_status=$?
7979

8080
DATE=$(date +'%D %r')

‎ci/scripts/driver.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,15 @@ esac
4747
######################################################
4848
# setup runtime env for correct python install and git
4949
######################################################
50+
HOMEgfs=${ROOT_DIR}
51+
export HOMEgfs
5052
set +x
5153
source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
5254
source "${ROOT_DIR}/ush/module-setup.sh"
5355
module use "${ROOT_DIR}/modulefiles"
5456
module load "module_gwsetup.${MACHINE_ID}"
5557
set -x
58+
unset HOMEgfs
5659

5760
############################################################
5861
# query repo and get list of open PRs with tags {machine}-CI

‎ci/scripts/run_ci.sh

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ set -eux
99
# Abstract TODO
1010
#####################################################################################
1111

12-
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
12+
HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
1313
scriptname=$(basename "${BASH_SOURCE[0]}")
1414
echo "Begin ${scriptname} at $(date -u)" || true
1515
export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -18,20 +18,21 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
1818
# Set up runtime environment varibles for accounts on supproted machines
1919
#########################################################################
2020

21-
source "${ROOT_DIR}/ush/detect_machine.sh"
21+
source "${HOMEgfs}/ush/detect_machine.sh"
2222
case ${MACHINE_ID} in
2323
hera | orion | hercules)
2424
echo "Running Automated Testing on ${MACHINE_ID}"
25-
source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
25+
source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
2626
;;
2727
*)
2828
echo "Unsupported platform. Exiting with error."
2929
exit 1
3030
;;
3131
esac
3232
set +x
33-
source "${ROOT_DIR}/ush/module-setup.sh"
34-
module use "${ROOT_DIR}/modulefiles"
33+
export HOMEgfs
34+
source "${HOMEgfs}/ush/module-setup.sh"
35+
module use "${HOMEgfs}/modulefiles"
3536
module load "module_gwsetup.${MACHINE_ID}"
3637
module list
3738
set -eux
@@ -47,7 +48,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
4748

4849
pr_list=""
4950
if [[ -f "${pr_list_dbfile}" ]]; then
50-
pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
51+
pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
5152
fi
5253
if [[ -z "${pr_list}" ]]; then
5354
echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting"

‎docs/source/clone.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ For coupled cycling (include new UFSDA) use the `-gu` options during build:
3939
./build_all.sh -gu
4040

4141

42+
For building with PDLIB for the wave model, use the `-w` options during build:
43+
44+
::
45+
46+
./build_all.sh -w
47+
48+
4249
Build workflow components and link workflow artifacts such as executables, etc.
4350

4451
::

‎env/WCOSS2.env

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,8 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
176176
if [[ "${step}" = "fcst" ]]; then
177177
export OMP_PLACES=cores
178178
export OMP_STACKSIZE=2048M
179+
export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
180+
export FI_OFI_RXM_SAR_LIMIT=3145728
179181
elif [[ "${step}" = "efcs" ]]; then
180182
export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
181183
export FI_OFI_RXM_SAR_LIMIT=3145728

‎parm/config/gfs/config.base.emc.dyn

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ export FIXcice="${FIXgfs}/cice"
3939
export FIXmom="${FIXgfs}/mom6"
4040
export FIXreg2grb2="${FIXgfs}/reg2grb2"
4141
export FIXugwd="${FIXgfs}/ugwd"
42+
export FIXgdas="${FIXgfs}/gdas"
4243

4344
########################################################################
4445

@@ -182,16 +183,16 @@ export ICERES="${OCNRES}"
182183
# These are the currently recommended grid-combinations
183184
case "${CASE}" in
184185
"C48")
185-
export waveGRD='glo_500'
186+
export waveGRD='uglo_100km'
186187
;;
187188
"C96" | "C192")
188-
export waveGRD='glo_200'
189+
export waveGRD='uglo_100km'
189190
;;
190191
"C384")
191-
export waveGRD='glo_025'
192+
export waveGRD='uglo_100km'
192193
;;
193194
"C768" | "C1152")
194-
export waveGRD='mx025'
195+
export waveGRD='uglo_m1g16'
195196
;;
196197
*)
197198
echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"
@@ -396,4 +397,11 @@ export FITSARC="YES"
396397
export FHMAX_FITS=132
397398
[[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS}
398399

400+
# The monitor jobs are not yet supported for JEDIATMVAR
401+
if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
402+
export DO_VERFOZN="NO" # Ozone data assimilation monitoring
403+
export DO_VERFRAD="NO" # Radiance data assimilation monitoring
404+
export DO_VMINMON="NO" # GSI minimization monitoring
405+
fi
406+
399407
echo "END: config.base"

‎parm/config/gfs/config.esfc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,9 @@ if [ $DOIAU_ENKF = "YES" ]; then
1616
export DOSFCANL_ENKF="NO"
1717
fi
1818

19+
# Turn off NST in JEDIATMENS
20+
if [[ "${DO_JEDIATMENS}" == "YES" ]]; then
21+
export DONST="NO"
22+
fi
23+
1924
echo "END: config.esfc"

‎parm/config/gfs/config.nsst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ echo "BEGIN: config.nsst"
1010
# nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled
1111
export NST_MODEL=2
1212

13+
# Set NST_MODEL for JEDIATMVAR or JEDIATMENS
14+
if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then
15+
export NST_MODEL=1
16+
fi
17+
1318
# nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON,
1419
export NST_SPINUP=0
1520
cdate="${PDY}${cyc}"

‎parm/config/gfs/config.resources

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ case ${step} in
9191
;;
9292

9393
"prepatmiodaobs")
94-
export wtime_prepatmiodaobs="00:10:00"
94+
export wtime_prepatmiodaobs="00:30:00"
9595
export npe_prepatmiodaobs=1
9696
export nth_prepatmiodaobs=1
9797
export npe_node_prepatmiodaobs=$(( npe_node_max / nth_prepatmiodaobs ))
@@ -196,8 +196,8 @@ case ${step} in
196196

197197
"atmanlinit")
198198
# make below case dependent later
199-
export layout_x=1
200-
export layout_y=1
199+
export layout_x=8
200+
export layout_y=8
201201

202202
export layout_gsib_x=$(( layout_x * 3 ))
203203
export layout_gsib_y=$(( layout_y * 2 ))
@@ -212,15 +212,16 @@ case ${step} in
212212

213213
"atmanlrun")
214214
# make below case dependent later
215-
export layout_x=1
216-
export layout_y=1
215+
export layout_x=8
216+
export layout_y=8
217217

218218
export wtime_atmanlrun="00:30:00"
219219
export npe_atmanlrun=$(( layout_x * layout_y * 6 ))
220220
export npe_atmanlrun_gfs=$(( layout_x * layout_y * 6 ))
221221
export nth_atmanlrun=1
222222
export nth_atmanlrun_gfs=${nth_atmanlrun}
223223
export npe_node_atmanlrun=$(( npe_node_max / nth_atmanlrun ))
224+
export memory_atmanlrun="96GB"
224225
export is_exclusive=True
225226
;;
226227

@@ -737,15 +738,15 @@ case ${step} in
737738
export npe_genesis=1
738739
export nth_genesis=1
739740
export npe_node_genesis=1
740-
export memory_genesis="4G"
741+
export memory_genesis="10G"
741742
;;
742743

743744
"genesis_fsu")
744745
export wtime_genesis_fsu="00:10:00"
745746
export npe_genesis_fsu=1
746747
export nth_genesis_fsu=1
747748
export npe_node_genesis_fsu=1
748-
export memory_genesis_fsu="4G"
749+
export memory_genesis_fsu="10G"
749750
;;
750751

751752
"fit2obs")
@@ -829,8 +830,8 @@ case ${step} in
829830

830831
"atmensanlinit")
831832
# make below case dependent later
832-
export layout_x=1
833-
export layout_y=1
833+
export layout_x=8
834+
export layout_y=8
834835

835836
export wtime_atmensanlinit="00:10:00"
836837
export npe_atmensanlinit=1
@@ -841,15 +842,16 @@ case ${step} in
841842

842843
"atmensanlrun")
843844
# make below case dependent later
844-
export layout_x=1
845-
export layout_y=1
845+
export layout_x=8
846+
export layout_y=8
846847

847848
export wtime_atmensanlrun="00:30:00"
848849
export npe_atmensanlrun=$(( layout_x * layout_y * 6 ))
849850
export npe_atmensanlrun_gfs=$(( layout_x * layout_y * 6 ))
850851
export nth_atmensanlrun=1
851852
export nth_atmensanlrun_gfs=${nth_atmensanlrun}
852853
export npe_node_atmensanlrun=$(( npe_node_max / nth_atmensanlrun ))
854+
export memory_atmensanlrun="96GB"
853855
export is_exclusive=True
854856
;;
855857

‎parm/config/gfs/config.sfcanl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,9 @@ echo "BEGIN: config.sfcanl"
88
# Get task specific resources
99
. $EXPDIR/config.resources sfcanl
1010

11+
# Turn off NST in JEDIATMVAR
12+
if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then
13+
export DONST="NO"
14+
fi
15+
1116
echo "END: config.sfcanl"

0 commit comments

Comments
 (0)