Skip to content

Commit 4eb296f

Browse files
RussTreadon-NOAARussTreadon-NOAA
authored
Optimize DA clone, build, and link (#931)
The PR contains changes to optimize the DA clone, build, and link. Changes are made to `checkout.sh`, `build_all.sh`, and `link_workflow.sh` in the g-w `sorc/` directory. These changes are in g-w branch `feature/clone` Two arguments are added to `checkout.sh` to allow the user to specify which DA package to build the global workflow with. These options are - `-g`: clone from the [GSI](https://github.com/NOAA-EMC/GSI) repo and build the g-w for GSI-based DA - `-u`: clone from the [GDASApp](https://github.com/NOAA-EMC/GDASApp) repo and build the g-w for UFS-based DA If no option is specified, `checkout.sh` does not clone any DA and DA related repos. This is the default behavior of `checkout.sh`. (_DA related_ repos include [GLDAS](https://github.com/NOAA-EMC/GLDAS), [GSI-utils](https://github.com/NOAA-EMC/GSI-utils), and [GSI-Monitor](https://github.com/NOAA-EMC/GSI-Monitor).) `build_all.sh` is modified to detect which repos and have been cloned and to build accordingly. `link_workflow.sh` is modified to detect which directories are present and link/copy accordingly. Closes #930
1 parent 2cad536 commit 4eb296f

File tree

3 files changed

+184
-89
lines changed

3 files changed

+184
-89
lines changed

‎sorc/build_all.sh‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -120,9 +120,10 @@ $Build_ufs_model && {
120120
}
121121

122122
#------------------------------------
123-
# build GSI and EnKF
123+
# build GSI and EnKF - optional checkout
124124
#------------------------------------
125-
$Build_gsi_enkf && {
125+
if [ -d gsi_enkf.fd ]; then
126+
$Build_gsi_enkf && {
126127
echo " .... Building gsi and enkf .... "
127128
./build_gsi_enkf.sh $_ops_opt $_verbose_opt > $logs_dir/build_gsi_enkf.log 2>&1
128129
rc=$?
@@ -132,11 +133,15 @@ $Build_gsi_enkf && {
132133
fi
133134
((err+=$rc))
134135
}
136+
else
137+
echo " .... Skip building gsi and enkf .... "
138+
fi
135139

136140
#------------------------------------
137141
# build gsi utilities
138142
#------------------------------------
139-
$Build_gsi_utils && {
143+
if [ -d gsi_utils.fd ]; then
144+
$Build_gsi_utils && {
140145
echo " .... Building gsi utilities .... "
141146
./build_gsi_utils.sh $_ops_opt $_verbose_opt > $logs_dir/build_gsi_utils.log 2>&1
142147
rc=$?
@@ -146,25 +151,33 @@ $Build_gsi_utils && {
146151
fi
147152
((err+=$rc))
148153
}
154+
else
155+
echo " .... Skip building gsi utilities .... "
156+
fi
149157

150158
#------------------------------------
151-
# build gdas
152-
#------------------------------------
153-
$Build_gdas && {
154-
echo " .... Building GDASApp .... "
155-
./build_gdas.sh > $logs_dir/build_gdas.log 2>&1
156-
rc=$?
157-
if [[ $rc -ne 0 ]] ; then
158-
echo "Fatal error in building GDAS."
159-
echo "The log file is in $logs_dir/build_gdas.log"
160-
fi
161-
((err+=$rc))
159+
# build gdas - optional checkout
160+
#------------------------------------
161+
if [ -d gdas.cd ]; then
162+
$Build_gdas && {
163+
echo " .... Building GDASApp .... "
164+
./build_gdas.sh $_verbose_opt > $logs_dir/build_gdas.log 2>&1
165+
rc=$?
166+
if [[ $rc -ne 0 ]] ; then
167+
echo "Fatal error in building GDASApp."
168+
echo "The log file is in $logs_dir/build_gdas.log"
169+
fi
170+
((err+=$rc))
162171
}
172+
else
173+
echo " .... Skip building GDASApp .... "
174+
fi
163175

164176
#------------------------------------
165177
# build gsi monitor
166178
#------------------------------------
167-
$Build_gsi_monitor && {
179+
if [ -d gsi_monitor.fd ]; then
180+
$Build_gsi_monitor && {
168181
echo " .... Building gsi monitor .... "
169182
./build_gsi_monitor.sh $_ops_opt $_verbose_opt > $logs_dir/build_gsi_monitor.log 2>&1
170183
rc=$?
@@ -174,6 +187,9 @@ $Build_gsi_monitor && {
174187
fi
175188
((err+=$rc))
176189
}
190+
else
191+
echo " .... Skip building gsi monitor .... "
192+
fi
177193

178194
#------------------------------------
179195
# build UPP
@@ -206,7 +222,8 @@ $Build_ufs_utils && {
206222
#------------------------------------
207223
# build gldas
208224
#------------------------------------
209-
$Build_gldas && {
225+
if [ -d gldas.fd ]; then
226+
$Build_gldas && {
210227
echo " .... Building gldas .... "
211228
./build_gldas.sh $_verbose_opt > $logs_dir/build_gldas.log 2>&1
212229
rc=$?
@@ -216,6 +233,9 @@ $Build_gldas && {
216233
fi
217234
((err+=$rc))
218235
}
236+
else
237+
echo " .... Skip building gldas .... "
238+
fi
219239

220240
#------------------------------------
221241
# build gfs_wafs - optional checkout

‎sorc/checkout.sh‎

Lines changed: 43 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ Usage: $BASH_SOURCE [-c][-h][-m ufs_hash][-o]
1818
Check out this UFS hash instead of the default
1919
-o:
2020
Check out operational-only code (GTG and WAFS)
21+
-g:
22+
Check out GSI for GSI-based DA
23+
-u:
24+
Check out GDASApp for UFS-based DA
2125
EOF
2226
exit 1
2327
}
@@ -56,7 +60,7 @@ function checkout() {
5660
fi
5761

5862
cd "${topdir}"
59-
if [[ -d "${dir}" && "${CLEAN:-NO}" == "YES" ]]; then
63+
if [[ -d "${dir}" && $CLEAN == "YES" ]]; then
6064
echo "|-- Removing existing clone in ${dir}"
6165
rm -Rf "$dir"
6266
fi
@@ -96,13 +100,30 @@ function checkout() {
96100
return 0
97101
}
98102

99-
while getopts ":chm:o" option; do
103+
# Set defaults for variables toggled by options
104+
export CLEAN="NO"
105+
CHECKOUT_GSI="NO"
106+
CHECKOUT_GDAS="NO"
107+
checkout_gtg="NO"
108+
checkout_wafs="NO"
109+
ufs_model_hash="b97375c"
110+
111+
# Parse command line arguments
112+
while getopts ":chgum:o" option; do
100113
case $option in
101114
c)
102115
echo "Recieved -c flag, will delete any existing directories and start clean"
103116
export CLEAN="YES"
104117
;;
118+
g)
119+
echo "Receieved -g flag for optional checkout of GSI-based DA"
120+
CHECKOUT_GSI="YES"
121+
;;
105122
h) usage;;
123+
u)
124+
echo "Received -u flag for optional checkout of UFS-based DA"
125+
CHECKOUT_GDAS="YES"
126+
;;
106127
o)
107128
echo "Received -o flag for optional checkout of operational-only codes"
108129
checkout_gtg="YES"
@@ -130,20 +151,29 @@ mkdir -p ${logdir}
130151

131152
# The checkout version should always be a speciifc commit (hash or tag), not a branch
132153
errs=0
133-
checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-b97375c}" ; errs=$((errs + $?))
134-
checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4" ; errs=$((errs + $?))
135-
checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "322cc7b" ; errs=$((errs + $?))
136-
checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "acf8870" ; errs=$((errs + $?))
137-
checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d" ; errs=$((errs + $?))
138-
checkout "gldas.fd" "https://github.com/NOAA-EMC/GLDAS.git" "fd8ba62" ; errs=$((errs + $?))
139-
checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "a2b0817" ; errs=$((errs + $?))
140-
checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?))
141-
142-
if [[ "${checkout_wafs:-NO}" == "YES" ]]; then
154+
checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash}"; errs=$((errs + $?))
155+
checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "a2b0817" ; errs=$((errs + $?))
156+
checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?))
157+
158+
if [[ $CHECKOUT_GSI == "YES" ]]; then
159+
checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4"; errs=$((errs + $?))
160+
fi
161+
162+
if [[ $CHECKOUT_GDAS == "YES" ]]; then
163+
checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d"; errs=$((errs + $?))
164+
fi
165+
166+
if [[ $CHECKOUT_GSI == "YES" || $CHECKOUT_GDAS == "YES" ]]; then
167+
checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "322cc7b"; errs=$((errs + $?))
168+
checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "acf8870"; errs=$((errs + $?))
169+
checkout "gldas.fd" "https://github.com/NOAA-EMC/GLDAS.git" "fd8ba62"; errs=$((errs + $?))
170+
fi
171+
172+
if [[ $checkout_wafs == "YES" ]]; then
143173
checkout "gfs_wafs.fd" "https://github.com/NOAA-EMC/EMC_gfs_wafs.git" "014a0b8"; errs=$((errs + $?))
144174
fi
145175

146-
if [[ "${checkout_gtg:-NO}" == "YES" ]]; then
176+
if [[ $checkout_gtg == "YES" ]]; then
147177
################################################################################
148178
# checkout_gtg
149179
## yes: The gtg code at NCAR private repository is available for ops. GFS only.

0 commit comments

Comments
 (0)