Merge pull request #1 from JessicaMeixner-NOAA/hr3pdlib

Add unstructured grid updates
JessicaMeixner-NOAA · Oct 17, 2023 · e73457d · e73457d
2 parents fdb4e99 + f43621c
commit e73457d
Show file tree

Hide file tree

Showing 5 changed files with 334 additions and 6 deletions.
diff --git a/tests/rt.conf b/tests/rt.conf
@@ -27,6 +27,15 @@ RUN | cpld_control_gfsv17                               |
 RUN | cpld_control_gfsv17_ugwpv1                        |                                    | baseline |
 RUN | cpld_control_gfsv17_iau                           |                                    | baseline | cpld_control_gfsv17
 
+#GFS
+COMPILE | s2swa_32bit_pdlib  | intel | -DAPP=S2SWA -D32BIT=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8_ugwpv1 -DPDLIB=ON | + hera orion cheyenne wcoss2 | fv3 |
+RUN | cpld_control_gfsv17_ugwpv1_pdlib                  | + hera orion cheyenne wcoss2    | baseline |
+RUN | cpld_restart_gfsv17_ugwpv1_pdlib                  | + hera orion cheyenne wcoss2    |          | cpld_control_gfsv17_ugwpv1_pdlib 
+RUN | cpld_mpi_gfsv17_ugwpv1_pdlib                      | + hera orion cheyenne wcoss2    |          | cpld_control_gfsv17_ugwpv1_pdlib
+
+COMPILE | s2swa_32bit_pdlib_debug | intel | -DAPP=S2SWA -D32BIT=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8_ugwpv1 -DPDLIB=ON -DDEBUG=ON | + hera orion cheyenne wcoss2 | fv3 |
+RUN | cpld_debug_gfsv17_ugwpv1_pdlib                    | + hera orion cheyenne wcoss2    | baseline |
+
 COMPILE | s2swa | intel | -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 | | fv3 |
 RUN | cpld_control_p8                                   |   - noaacloud                       | baseline |
 RUN | cpld_restart_p8                                   |   - noaacloud                       |          | cpld_control_p8

diff --git a/tests/tests/cpld_control_gfsv17_ugwpv1 → tests/tests/cpld_control_gfsv17_ugwpv1_pdlib b/tests/tests/cpld_control_gfsv17_ugwpv1 → tests/tests/cpld_control_gfsv17_ugwpv1_pdlib
@@ -3,9 +3,9 @@
 #  version 1 non-stationary GWD
 #
 
-export TEST_DESCR="Fully coupled FV3-CCPP-MOM6-CICE-CMEPS-WW3 system - C96MX100"
+export TEST_DESCR="Fully coupled FV3-CCPP-MOM6-CICE-CMEPS-WW3 system - C96MX100 - unst WW3"
 
-export CNTL_DIR="cpld_control_gfsv17_ugwpv1"
+export CNTL_DIR="cpld_control_gfsv17_ugwpv1_pdlib"
 
 export LIST_FILES="sfcf021.tile1.nc \
                    sfcf021.tile2.nc \
@@ -79,6 +79,17 @@ export RESTART_N=12
 export RESTART_INTERVAL="${RESTART_N} -1"
 export OUTPUT_FH='0 21 24'
 
+export INPES=$INPES_cpl_unstr
+export JNPES=$JNPES_cpl_unstr
+export WRTTASK_PER_GROUP=$(( WPG_cpl_unstr * THRD_cpl_unstr ))
+
+OCN_tasks=$OCN_tasks_cpl_unstr
+ICE_tasks=$ICE_tasks_cpl_unstr
+WAV_tasks=$WAV_tasks_cpl_unstr
+
+export atm_omp_num_threads=$THRD_cpl_unstr
+export med_omp_num_threads=$atm_omp_num_threads
+
 # P8 UGWP1
 export GWD_OPT=2
 export CDMBWD=20.0,2.5,1.0,1.0
@@ -97,6 +108,10 @@ export TOPOEDITS=ufs.topo_edits_011818.nc
 export MOM6_ALLOW_LANDMASK_CHANGES=True
 export eps_imesh=2.5e-1
 
+export WAVDOMAIN=global_270k
+export MESH_WAV=mesh.${WAVDOMAIN}.nc
+export MODDEF_WAV=mod_def.exp.${WAVDOMAIN}
+
 export FV3_RUN=cpld_control_run.IN
 export CCPP_SUITE=FV3_GFS_v17_coupled_p8_ugwpv1
 export FIELD_TABLE=field_table_thompson_noaero_tke_progsigma
@@ -110,3 +125,7 @@ export PROGSIGMA=.true.
 if [[ $MACHINE_ID = cheyenne ]]; then
   TPN=18
 fi
+
+if [[ $MACHINE_ID = hera ]] && [[ $RT_COMPILER = gnu ]]; then
+  WLCLK=40
+fi
diff --git a/tests/tests/cpld_debug_gfsv17_ugwpv1_pdlib b/tests/tests/cpld_debug_gfsv17_ugwpv1_pdlib
@@ -0,0 +1,123 @@
+#
+#  cpld_debug unstrWW3 PDLIB GFSv17 test
+#
+
+export TEST_DESCR="Fully coupled FV3-CCPP-MOM6-CICE-CMEPS-WW3 system -C96MX100 unstr WW3 PDLIB - GFSv17 debug"
+
+export CNTL_DIR="cpld_debug_gfsv17_ugwpv1_pdlib"
+
+export LIST_FILES="sfcf003.tile1.nc \
+                   sfcf003.tile2.nc \
+                   sfcf003.tile3.nc \
+                   sfcf003.tile4.nc \
+                   sfcf003.tile5.nc \
+                   sfcf003.tile6.nc \
+                   atmf003.tile1.nc \
+                   atmf003.tile2.nc \
+                   atmf003.tile3.nc \
+                   atmf003.tile4.nc \
+                   atmf003.tile5.nc \
+                   atmf003.tile6.nc \
+                   RESTART/20210322.090000.coupler.res \
+                   RESTART/20210322.090000.fv_core.res.nc \
+                   RESTART/20210322.090000.fv_core.res.tile1.nc \
+                   RESTART/20210322.090000.fv_core.res.tile2.nc \
+                   RESTART/20210322.090000.fv_core.res.tile3.nc \
+                   RESTART/20210322.090000.fv_core.res.tile4.nc \
+                   RESTART/20210322.090000.fv_core.res.tile5.nc \
+                   RESTART/20210322.090000.fv_core.res.tile6.nc \
+                   RESTART/20210322.090000.fv_srf_wnd.res.tile1.nc \
+                   RESTART/20210322.090000.fv_srf_wnd.res.tile2.nc \
+                   RESTART/20210322.090000.fv_srf_wnd.res.tile3.nc \
+                   RESTART/20210322.090000.fv_srf_wnd.res.tile4.nc \
+                   RESTART/20210322.090000.fv_srf_wnd.res.tile5.nc \
+                   RESTART/20210322.090000.fv_srf_wnd.res.tile6.nc \
+                   RESTART/20210322.090000.fv_tracer.res.tile1.nc \
+                   RESTART/20210322.090000.fv_tracer.res.tile2.nc \
+                   RESTART/20210322.090000.fv_tracer.res.tile3.nc \
+                   RESTART/20210322.090000.fv_tracer.res.tile4.nc \
+                   RESTART/20210322.090000.fv_tracer.res.tile5.nc \
+                   RESTART/20210322.090000.fv_tracer.res.tile6.nc \
+                   RESTART/20210322.090000.phy_data.tile1.nc \
+                   RESTART/20210322.090000.phy_data.tile2.nc \
+                   RESTART/20210322.090000.phy_data.tile3.nc \
+                   RESTART/20210322.090000.phy_data.tile4.nc \
+                   RESTART/20210322.090000.phy_data.tile5.nc \
+                   RESTART/20210322.090000.phy_data.tile6.nc \
+                   RESTART/20210322.090000.sfc_data.tile1.nc \
+                   RESTART/20210322.090000.sfc_data.tile2.nc \
+                   RESTART/20210322.090000.sfc_data.tile3.nc \
+                   RESTART/20210322.090000.sfc_data.tile4.nc \
+                   RESTART/20210322.090000.sfc_data.tile5.nc \
+                   RESTART/20210322.090000.sfc_data.tile6.nc \
+                   RESTART/20210322.090000.MOM.res.nc \
+                   RESTART/iced.2021-03-22-32400.nc \
+                   RESTART/ufs.cpld.cpl.r.2021-03-22-32400.nc \
+                   20210322.090000.out_pnt.ww3 \
+                   20210322.090000.out_grd.ww3 "
+
+export_fv3
+export_cpl
+
+export FHMAX=3
+export DAYS=0.125
+
+export RESTART_N=${FHMAX}
+export OUTPUT_FH="3 -1"
+
+export INPES=$INPES_cpl_unstr
+export JNPES=$JNPES_cpl_unstr
+export WRTTASK_PER_GROUP=$(( WPG_cpl_unstr * THRD_cpl_unstr ))
+
+OCN_tasks=$OCN_tasks_cpl_unstr
+ICE_tasks=$ICE_tasks_cpl_unstr
+WAV_tasks=$WAV_tasks_cpl_unstr
+# bump resources for debug test
+WAV_tasks="$(($WAV_tasks_cpl_unstr + 18))"
+
+export atm_omp_num_threads=$THRD_cpl_unstr
+export med_omp_num_threads=$atm_omp_num_threads
+
+export CPLCHM=.false.
+export DNATS=0
+export TOPOEDITS=ufs.topo_edits_011818.nc
+export MOM6_ALLOW_LANDMASK_CHANGES=True
+export eps_imesh=2.5e-1
+export DIAG_FREQ=`expr $((FHMAX-$FHROT)) \* 3600 / $DT_CICE`
+
+export GWD_OPT=2
+export CDMBWD=20.0,2.5,1.0,1.0
+export DO_UGWP_V1=.true.
+export KNOB_UGWP_VERSION=1
+export KNOB_UGWP_NSLOPE=1
+export DO_UGWP_V0=.false.
+export DO_GSL_DRAG_LS_BL=.true.
+export DO_GSL_DRAG_SS=.true.
+export DO_GSL_DRAG_TOFD=.true.
+export DO_UGWP_V1_OROG_ONLY=.false.
+export DO_UGWP_V0_NST_ONLY=.false.
+export LDIAG_UGWP=.false.
+
+#GFSv17 settings
+export PROGSIGMA=.true.
+
+export WAVDOMAIN=global_270k
+export MESH_WAV=mesh.${WAVDOMAIN}.nc
+export MODDEF_WAV=mod_def.exp.${WAVDOMAIN}
+
+export FV3_RUN=cpld_control_run.IN
+export CCPP_SUITE=FV3_GFS_v17_coupled_p8_ugwpv1
+export FIELD_TABLE=field_table_thompson_noaero_tke_progsigma
+export DIAG_TABLE=diag_table_cpld_template
+export INPUT_NML=cpld_control_ugwpv1.nml.IN
+export NEMS_CONFIGURE=nems.configure.cpld_noaero_outwav.IN
+
+export FV3_RUN=cpld_control_run.IN
+
+if [[ $MACHINE_ID = cheyenne ]]; then
+  TPN=18
+fi
+
+if [[ $MACHINE_ID = hera ]] && [[ $RT_COMPILER = gnu ]]; then
+  WLCLK=40
+fi
diff --git a/tests/tests/cpld_control_gfsv17 → tests/tests/cpld_mpi_gfsv17_ugwpv1_pdlib b/tests/tests/cpld_control_gfsv17 → tests/tests/cpld_mpi_gfsv17_ugwpv1_pdlib
@@ -1,10 +1,11 @@
 #
-#  cpld_control GFSv17 test
+#  cpld_control GFSv17 with UGWPv1 test, i.e., GSL orographic drag + 
+#  version 1 non-stationary GWD
 #
 
-export TEST_DESCR="Fully coupled FV3-CCPP-MOM6-CICE-CMEPS-WW3 system - C96MX100"
+export TEST_DESCR="Fully coupled FV3-CCPP-MOM6-CICE-CMEPS-WW3 system - C96MX100 - unst WW3"
 
-export CNTL_DIR="cpld_control_gfsv17"
+export CNTL_DIR="cpld_control_gfsv17_ugwpv1_pdlib"
 
 export LIST_FILES="sfcf021.tile1.nc \
                    sfcf021.tile2.nc \
@@ -78,14 +79,53 @@ export RESTART_N=3
 export RESTART_INTERVAL="${RESTART_N} -1"
 export OUTPUT_FH='0 21 24'
 
+
+export INPES=$INPES_cpl_unstr_mpi
+export JNPES=$JNPES_cpl_unstr_mpi
+export atm_omp_num_threads=$THRD_cpl_unstr_mpi
+export WRTTASK_PER_GROUP=$(( WPG_cpl_unstr_mpi * THRD_cpl_unstr_mpi ))
+
+OCN_tasks=$OCN_tasks_cpl_unstr_mpi
+ICE_tasks=$ICE_tasks_cpl_unstr_mpi
+WAV_tasks=$WAV_tasks_cpl_unstr_mpi
+
+export NPROC_ICE=$ICE_tasks
+export np2=`expr $NPROC_ICE / 2`
+export BLCKX=`expr $NX_GLB / $np2`
+export BLCKY=`expr $NY_GLB / 2`
+
+
+
+export atm_omp_num_threads=$THRD_cpl_unstr
+export med_omp_num_threads=$atm_omp_num_threads
+
+# P8 UGWP1
+export GWD_OPT=2
+export CDMBWD=20.0,2.5,1.0,1.0
+export DO_UGWP_V1=.true.
+export KNOB_UGWP_VERSION=1
+export KNOB_UGWP_NSLOPE=1
+export DO_UGWP_V0=.false.
+export DO_GSL_DRAG_LS_BL=.true.
+export DO_GSL_DRAG_SS=.true.
+export DO_GSL_DRAG_TOFD=.true.
+export DO_UGWP_V1_OROG_ONLY=.false.
+export DO_UGWP_V0_NST_ONLY=.false.
+export LDIAG_UGWP=.false.
+
 export TOPOEDITS=ufs.topo_edits_011818.nc
 export MOM6_ALLOW_LANDMASK_CHANGES=True
 export eps_imesh=2.5e-1
 
+export WAVDOMAIN=global_270k
+export MESH_WAV=mesh.${WAVDOMAIN}.nc
+export MODDEF_WAV=mod_def.exp.${WAVDOMAIN}
+
 export FV3_RUN=cpld_control_run.IN
+export CCPP_SUITE=FV3_GFS_v17_coupled_p8_ugwpv1
 export FIELD_TABLE=field_table_thompson_noaero_tke_progsigma
 export DIAG_TABLE=diag_table_cpld_template
-export INPUT_NML=cpld_control.nml.IN
+export INPUT_NML=cpld_control_ugwpv1.nml.IN
 export NEMS_CONFIGURE=nems.configure.cpld_noaero_outwav.IN
 
 #GFSv17	settings

diff --git a/tests/tests/cpld_restart_gfsv17_ugwpv1_pdlib b/tests/tests/cpld_restart_gfsv17_ugwpv1_pdlib
@@ -0,0 +1,137 @@
+#
+#  cpld_control GFSv17 with UGWPv1 test, i.e., GSL orographic drag + 
+#  version 1 non-stationary GWD
+#
+
+export TEST_DESCR="Fully coupled FV3-CCPP-MOM6-CICE-CMEPS-WW3 system - C96MX100 - unst WW3 - Restart "
+
+export CNTL_DIR="cpld_control_gfsv17_ugwpv1_pdlib"
+
+export LIST_FILES="sfcf024.tile1.nc \
+                   sfcf024.tile1.nc \
+                   sfcf024.tile2.nc \
+                   sfcf024.tile3.nc \
+                   sfcf024.tile4.nc \
+                   sfcf024.tile5.nc \
+                   sfcf024.tile6.nc \
+                   atmf024.tile1.nc \
+                   atmf024.tile2.nc \
+                   atmf024.tile3.nc \
+                   atmf024.tile4.nc \
+                   atmf024.tile5.nc \
+                   atmf024.tile6.nc \
+                   RESTART/20210323.060000.coupler.res \
+                   RESTART/20210323.060000.fv_core.res.nc \
+                   RESTART/20210323.060000.fv_core.res.tile1.nc \
+                   RESTART/20210323.060000.fv_core.res.tile2.nc \
+                   RESTART/20210323.060000.fv_core.res.tile3.nc \
+                   RESTART/20210323.060000.fv_core.res.tile4.nc \
+                   RESTART/20210323.060000.fv_core.res.tile5.nc \
+                   RESTART/20210323.060000.fv_core.res.tile6.nc \
+                   RESTART/20210323.060000.fv_srf_wnd.res.tile1.nc \
+                   RESTART/20210323.060000.fv_srf_wnd.res.tile2.nc \
+                   RESTART/20210323.060000.fv_srf_wnd.res.tile3.nc \
+                   RESTART/20210323.060000.fv_srf_wnd.res.tile4.nc \
+                   RESTART/20210323.060000.fv_srf_wnd.res.tile5.nc \
+                   RESTART/20210323.060000.fv_srf_wnd.res.tile6.nc \
+                   RESTART/20210323.060000.fv_tracer.res.tile1.nc \
+                   RESTART/20210323.060000.fv_tracer.res.tile2.nc \
+                   RESTART/20210323.060000.fv_tracer.res.tile3.nc \
+                   RESTART/20210323.060000.fv_tracer.res.tile4.nc \
+                   RESTART/20210323.060000.fv_tracer.res.tile5.nc \
+                   RESTART/20210323.060000.fv_tracer.res.tile6.nc \
+                   RESTART/20210323.060000.phy_data.tile1.nc \
+                   RESTART/20210323.060000.phy_data.tile2.nc \
+                   RESTART/20210323.060000.phy_data.tile3.nc \
+                   RESTART/20210323.060000.phy_data.tile4.nc \
+                   RESTART/20210323.060000.phy_data.tile5.nc \
+                   RESTART/20210323.060000.phy_data.tile6.nc \
+                   RESTART/20210323.060000.sfc_data.tile1.nc \
+                   RESTART/20210323.060000.sfc_data.tile2.nc \
+                   RESTART/20210323.060000.sfc_data.tile3.nc \
+                   RESTART/20210323.060000.sfc_data.tile4.nc \
+                   RESTART/20210323.060000.sfc_data.tile5.nc \
+                   RESTART/20210323.060000.sfc_data.tile6.nc \
+                   RESTART/20210323.060000.MOM.res.nc \
+                   RESTART/iced.2021-03-23-21600.nc \
+                   RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc \
+                   20210323.060000.out_pnt.ww3 \
+                   20210323.060000.out_grd.ww3 "
+
+export_fv3
+export_cpl
+
+export CPLCHM=.false.
+export DNATS=0
+
+export FHROT=12
+export RESTART_N=$((FHMAX-$FHROT))
+
+export RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( ${FHROT} + ${SHOUR} )))0000"
+export RESTART_FILE_SUFFIX_SECS="${SYEAR}-${SMONTH}-${SDAY}-$(printf "%05d" $(( (${FHROT}+ ${SHOUR})*3600 )))"
+
+# ATM warm start
+export WARM_START=.true.
+export MAKE_NH=.false.
+export NA_INIT=0
+export EXTERNAL_IC=.false.
+export NGGPS_IC=.false.
+export MOUNTAIN=.true.
+
+# ICE warm start
+export CICERUNTYPE=continue
+export RUNTYPE=continue
+export USE_RESTART_TIME=.true.
+
+# MOM6 warm start
+export MOM6_RESTART_SETTING=r
+
+
+export INPES=$INPES_cpl_unstr
+export JNPES=$JNPES_cpl_unstr
+export WRTTASK_PER_GROUP=$(( WPG_cpl_unstr * THRD_cpl_unstr ))
+
+OCN_tasks=$OCN_tasks_cpl_unstr
+ICE_tasks=$ICE_tasks_cpl_unstr
+WAV_tasks=$WAV_tasks_cpl_unstr
+
+export atm_omp_num_threads=$THRD_cpl_unstr
+export med_omp_num_threads=$atm_omp_num_threads
+
+# P8 UGWP1
+export GWD_OPT=2
+export CDMBWD=20.0,2.5,1.0,1.0
+export DO_UGWP_V1=.true.
+export KNOB_UGWP_VERSION=1
+export KNOB_UGWP_NSLOPE=1
+export DO_UGWP_V0=.false.
+export DO_GSL_DRAG_LS_BL=.true.
+export DO_GSL_DRAG_SS=.true.
+export DO_GSL_DRAG_TOFD=.true.
+export DO_UGWP_V1_OROG_ONLY=.false.
+export DO_UGWP_V0_NST_ONLY=.false.
+export LDIAG_UGWP=.false.
+
+export TOPOEDITS=ufs.topo_edits_011818.nc
+export MOM6_ALLOW_LANDMASK_CHANGES=True
+export eps_imesh=2.5e-1
+export DIAG_FREQ=`expr $((FHMAX-$FHROT)) \* 3600 / $DT_CICE`
+
+export WAVDOMAIN=global_270k
+export MESH_WAV=mesh.${WAVDOMAIN}.nc
+export MODDEF_WAV=mod_def.exp.${WAVDOMAIN}
+
+export FV3_RUN=cpld_control_run.IN
+export CCPP_SUITE=FV3_GFS_v17_coupled_p8_ugwpv1
+export FIELD_TABLE=field_table_thompson_noaero_tke_progsigma
+export DIAG_TABLE=diag_table_cpld_template
+export INPUT_NML=cpld_control_ugwpv1.nml.IN
+export NEMS_CONFIGURE=nems.configure.cpld_noaero_outwav.IN
+
+#GFSv17	settings
+export PROGSIGMA=.true.
+
+if [[ $MACHINE_ID = cheyenne ]]; then
+  TPN=18
+fi
+