Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update UFS to develop as of 2023 Apr 17 #1509

Merged
merged 13 commits into from
May 1, 2023
Merged

Update UFS to develop as of 2023 Apr 17 #1509

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
8ef27c1
Update UFS to develop as of 2023 Apr 17
WalterKolczynski-NOAA Apr 24, 2023
e50241c
Remove unused diag table
WalterKolczynski-NOAA Apr 25, 2023
1b58c7e
Undo change to DA diag table
WalterKolczynski-NOAA Apr 25, 2023
43e05cc
patch to PR1509 related to the write tasks per group update
aerorahul Apr 25, 2023
265e9e1
Merge branch 'develop' into patch/1509
aerorahul Apr 26, 2023
6196d70
make it clear write tasks are per thread and per tile
aerorahul Apr 26, 2023
1e3e8c7
Update UFS to develop as of 2023 Apr 17
WalterKolczynski-NOAA Apr 24, 2023
1910490
Remove unused diag table
WalterKolczynski-NOAA Apr 25, 2023
7c3b4b7
Undo change to DA diag table
WalterKolczynski-NOAA Apr 25, 2023
464e7f1
Merge branch 'feature/ufs_update' into patch/1509
WalterKolczynski-NOAA Apr 26, 2023
ce8117e
Merge pull request #25 from aerorahul/patch/1509
WalterKolczynski-NOAA Apr 26, 2023
c6198a3
Define WRTTASK_PER_GROUP_PER_THREAD
WalterKolczynski-NOAA Apr 26, 2023
94442ed
Change write tasks back to per thread
WalterKolczynski-NOAA May 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Externals.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# External sub-modules of global-workflow

[UFS]
tag = c22aaad
tag = 2247060
local_path = sorc/ufs_model.fd
repo_url = https://github.com/ufs-community/ufs-weather-model.git
protocol = git
Expand Down
2 changes: 1 addition & 1 deletion jobs/JGLOBAL_ATMOS_POST
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ fi
# Specify Restart File Name to Key Off
#######################################
# TODO Improve the name of this variable
export restart_file=${COM_ATMOS_HISTORY}/${RUN}.t${cyc}z.logf
export restart_file=${COM_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf

####################################
# Specify Timeout Behavior of Post
Expand Down
5 changes: 2 additions & 3 deletions parm/config/config.resources
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then
export layout_x=${layout_x_gfs}
export layout_y=${layout_y_gfs}
export WRITE_GROUP=${WRITE_GROUP_GFS}
export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP_GFS}
export WRTTASK_PER_GROUP_PER_THREAD=${WRTTASK_PER_GROUP_PER_THREAD_GFS}
ntasks_fv3=${ntasks_fv3_gfs}
ntasks_quilt=${ntasks_quilt_gfs}
nthreads_fv3=${nthreads_fv3_gfs}
Expand All @@ -530,8 +530,7 @@ elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then
# PETS for quilting
if [[ "${QUILTING:-}" = ".true." ]]; then
(( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
(( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP * nthreads_fv3 )) # when threads are used, WRTTASKS_PER_GROUP = INCOMING_WRTTASKS_PER_GROUP * threads
# model_configure should be updated to reflect the number of threads used by calling this variable write_pets_per_group
(( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD * nthreads_fv3 ))
export WRTTASK_PER_GROUP
else
QUILTPETS=0
Expand Down
37 changes: 21 additions & 16 deletions parm/config/config.ufs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,9 @@ case "${fv3_res}" in
export nthreads_fv3_gfs=1
export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=2
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
export WRITE_GROUP_GFS=1
export WRTTASK_PER_GROUP_GFS=2
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
;;
"C96")
export DELTIM=600
Expand All @@ -127,9 +127,9 @@ case "${fv3_res}" in
export nthreads_fv3_gfs=1
export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=4
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
export WRITE_GROUP_GFS=1
export WRTTASK_PER_GROUP_GFS=4
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
;;
"C192")
export DELTIM=450
Expand All @@ -141,9 +141,9 @@ case "${fv3_res}" in
export nthreads_fv3_gfs=2
export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=64
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
export WRITE_GROUP_GFS=2
export WRTTASK_PER_GROUP_GFS=64
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=5
;;
"C384")
export DELTIM=300
Expand All @@ -155,9 +155,9 @@ case "${fv3_res}" in
export nthreads_fv3_gfs=2
export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=2
export WRTTASK_PER_GROUP=48
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=8
export WRITE_GROUP_GFS=2
export WRTTASK_PER_GROUP_GFS=48
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=4
;;
"C768")
export DELTIM=150
Expand All @@ -169,9 +169,9 @@ case "${fv3_res}" in
export nthreads_fv3_gfs=4
export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=2
export WRTTASK_PER_GROUP=64
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
export WRITE_GROUP_GFS=4
export WRTTASK_PER_GROUP_GFS=64
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10
;;
"C1152")
export DELTIM=120
Expand All @@ -183,9 +183,9 @@ case "${fv3_res}" in
export nthreads_fv3_gfs=4
export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=4
export WRTTASK_PER_GROUP=64 # TODO: refine these numbers when a case is available
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available
export WRITE_GROUP_GFS=4
export WRTTASK_PER_GROUP_GFS=64 # TODO: refine these numbers when a case is available
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10 # TODO: refine these numbers when a case is available
;;
"C3072")
export DELTIM=90
Expand All @@ -197,23 +197,28 @@ case "${fv3_res}" in
export nthreads_fv3_gfs=4
export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=4
export WRTTASK_PER_GROUP=64 # TODO: refine these numbers when a case is available
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available
export WRITE_GROUP_GFS=4
export WRTTASK_PER_GROUP_GFS=64 # TODO: refine these numbers when a case is available
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10 # TODO: refine these numbers when a case is available
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we change this to a multiple of 6, e,g, 12? It gives developers option to try writing restart files on write grid comp?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As seen from the variable name, it is per tile. So, it gets multiplied by 6 before ending up in model_configure.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@junwang-noaa The meaning of the variable has already been changed to be per tile (we multiply by 6 later), so it is impossible for the end value to not be divisible by 6.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the explanation.

;;
*)
echo "FATAL ERROR: Unsupported FV3 resolution = ${fv3_res}, ABORT!"
exit 1
;;
esac

(( WRTTASK_PER_GROUP_PER_THREAD = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE * 6 ))
(( WRTTASK_PER_GROUP_PER_THREAD_GFS = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS * 6 ))
export WRTTASK_PER_GROUP_PER_THREAD
export WRTTASK_PER_GROUP_PER_THREAD_GFS

(( ntasks_fv3 = layout_x * layout_y * 6 ))
(( ntasks_fv3_gfs = layout_x_gfs * layout_y_gfs * 6 ))
export ntasks_fv3
export ntasks_fv3_gfs

(( ntasks_quilt = WRITE_GROUP * WRTTASK_PER_GROUP ))
(( ntasks_quilt_gfs = WRITE_GROUP_GFS * WRTTASK_PER_GROUP_GFS ))
(( ntasks_quilt = WRITE_GROUP * WRTTASK_PER_GROUP_PER_THREAD ))
(( ntasks_quilt_gfs = WRITE_GROUP_GFS * WRTTASK_PER_GROUP_PER_THREAD_GFS ))
export ntasks_quilt
export ntasks_quilt_gfs

Expand Down
1 change: 0 additions & 1 deletion parm/mom6/MOM_input_template_025
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,6 @@ DTFREEZE_DP = -7.75E-08 ! [deg C Pa-1] default = 0.0
PARALLEL_RESTARTFILES = True ! [Boolean] default = False
! If true, each processor writes its own restart file, otherwise a single
! restart file is generated
STORE_CORIOLIS_ACCEL = False

! === module MOM_tracer_flow_control ===
USE_IDEAL_AGE_TRACER = False ! [Boolean] default = False
Expand Down
1 change: 0 additions & 1 deletion parm/mom6/MOM_input_template_050
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,6 @@ DTFREEZE_DP = -7.75E-08 ! [deg C Pa-1] default = 0.0
PARALLEL_RESTARTFILES = True ! [Boolean] default = False
! If true, each processor writes its own restart file, otherwise a single
! restart file is generated
STORE_CORIOLIS_ACCEL = False

! === module MOM_tracer_flow_control ===
USE_IDEAL_AGE_TRACER = False ! [Boolean] default = False
Expand Down
1 change: 0 additions & 1 deletion parm/mom6/MOM_input_template_100
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,6 @@ TFREEZE_FORM = "MILLERO_78" ! default = "LINEAR"
PARALLEL_RESTARTFILES = True ! [Boolean] default = False
! If true, each processor writes its own restart file, otherwise a single
! restart file is generated
STORE_CORIOLIS_ACCEL = False

! === module MOM_tracer_flow_control ===
USE_IDEAL_AGE_TRACER = False ! [Boolean] default = False
Expand Down
1 change: 0 additions & 1 deletion parm/mom6/MOM_input_template_500
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ TFREEZE_FORM = "MILLERO_78" ! default = "LINEAR"

! === module MOM_restart ===
RESTART_CHECKSUMS_REQUIRED = False
STORE_CORIOLIS_ACCEL = False
! === module MOM_tracer_flow_control ===

! === module MOM_coord_initialization ===
Expand Down
7 changes: 7 additions & 0 deletions parm/parm_fv3diag/diag_table
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,13 @@
"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2

"gfs_phys", "cldfra", "cldfra", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "frzr", "frzr", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "frzrb", "frzrb", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "frozr", "frozr", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "frozrb", "frozrb", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "tsnowp", "tsnowp", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "tsnowpb", "tsnowpb", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "rhonewsn", "rhonewsn", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2
Expand Down
Loading