Update to HR1 (#1197)

Updates settings for the HR1 prototype Compilation is switched to "mixed-mode", with a 32-bit atmosphere and 64-bit for other components. Atmosphere physics options are updated. Changing the wave model to run on the outer loop and the resolution to ¼-deg. The model version was already updated previously, but this commit updates some of the associated input files.
NOAA-EMC · Mar 2, 2023 · 0c523d9 · 0c523d9
1 parent 5d6c71a
commit 0c523d9
Show file tree

Hide file tree

Showing 51 changed files with 638 additions and 55,706 deletions.
diff --git a/jobs/rocoto/vrfy.sh b/jobs/rocoto/vrfy.sh
@@ -31,7 +31,7 @@ export COMIN="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}"
 echo
 echo "=============== START TO GENERATE QUARTER DEGREE GRIB1 FILES ==============="
 if [ ${MKPGB4PRCP} = "YES" -a ${CDUMP} = "gfs" ]; then
-    if [ ! -d ${ARCDIR} ]; then mkdir ${ARCDIR} ; fi
+    if [ ! -d ${ARCDIR} ]; then mkdir -p ${ARCDIR} ; fi
     nthreads_env=${OMP_NUM_THREADS:-1} # get threads set in env
     export OMP_NUM_THREADS=1
     cd ${COMIN}

diff --git a/modulefiles/module_base.wcoss2.lua b/modulefiles/module_base.wcoss2.lua
@@ -3,7 +3,7 @@ Load environment to run GFS on WCOSS2
 ]])
 
 load(pathJoin("PrgEnv-intel", "8.1.0"))
-load(pathJoin("craype", "2.7.10"))
+load(pathJoin("craype", "2.7.13"))
 load(pathJoin("intel", "19.1.3.304"))
 load(pathJoin("cray-mpich", "8.1.9"))
 load(pathJoin("cray-pals", "1.0.17"))

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
@@ -170,15 +170,15 @@ export OPS_RES="C768" # Do not change
 export LEVS=128
 export CASE="@CASECTL@"
 export CASE_ENKF="@CASEENS@"
-case "$CASE" in
+case "${CASE}" in
     "C48") export OCNRES=500;;
     "C96") export OCNRES=100;;
     "C192") export OCNRES=050;;
     "C384") export OCNRES=025;;
     "C768") export OCNRES=025;;
     *) export OCNRES=025;;
 esac
-export ICERES=$OCNRES
+export ICERES=${OCNRES}
 
 case "${APP}" in
   ATM)
@@ -215,7 +215,7 @@ case "${APP}" in
         export DO_WAVE="YES"
         export WAVE_CDUMP="both"
         export cplwav2atm=".true."
-        export confignamevarfornems="${confignamevarfornems}_wave"
+        export confignamevarfornems="${confignamevarfornems}_outerwave"
     fi
 
     source ${EXPDIR}/config.defaults.s2sw

diff --git a/parm/config/config.coupled_ic b/parm/config/config.coupled_ic
@@ -8,7 +8,7 @@ echo "BEGIN: config.coupled_ic"
 source ${EXPDIR}/config.resources coupled_ic
 
 if [[ "${machine}" == "WCOSS2" ]]; then
-  export BASE_CPLIC="/lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED"
+  export BASE_CPLIC="/lfs/h2/emc/couple/noscrub/Jiande.Wang/IC"
 elif [[ "${machine}" == "HERA" ]]; then
   export BASE_CPLIC="/scratch1/NCEPDEV/climate/role.ufscpara/IC"
 elif [[ "${machine}" == "ORION" ]]; then
@@ -17,10 +17,25 @@ elif [[ "${machine}" == "S4" ]]; then
   export BASE_CPLIC="/data/prod/glopara/coupled_ICs"
 fi
 
-export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c
-export CPL_ICEIC=CPC
-export CPL_OCNIC=CPC3Dvar
-export CPL_WAVIC=GEFSwave20210528v2
-export CPL_DATM=CDEPS_DATM
+
+case "${CASE}" in
+  "C384")
+    #C384 and P8 ICs
+    export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c
+    export CPL_ICEIC=CPC
+    export CPL_OCNIC=CPC3Dvar
+    export CPL_WAVIC=GEFSwave20210528v2
+    ;;
+  "C768")
+    export CPL_ATMIC=HR1
+    export CPL_ICEIC=HR1
+    export CPL_OCNIC=HR1
+    export CPL_WAVIC=HR1
+    ;;
+  *)
+    echo "Unrecognized case: ${1}"
+    exit 1
+    ;;
+esac
 
 echo "END: config.coupled_ic"
diff --git a/parm/config/config.defaults.s2sw b/parm/config/config.defaults.s2sw
@@ -27,21 +27,22 @@ case "${CASE}" in
     #layout_y_gfs=16
     #WRTTASK_PER_GROUP_GFS=86
     WRTIOBUF="32M"  # TODO: This value is for P8 w/ C384.  Why not update/set this as default in config.fv3 C384?
-    MEDPETS=300  # TODO: P8 wants to use 300 instead of ATMPETS = layout_x * layout_y * 6
+    MEDPETS=300  # TODO: Use 300 instead of ATMPETS = layout_x * layout_y * 6
+    ;;
+  "C768")
+    MEDPETS=300  
     ;;
 esac
 
-# config.ice
-
-# TODO: These also are likely P8 hard-wired configurations.  Perhaps this file should be config.P8.defaults
 # config.wave
-waveGRD='gwes_30m'
-waveinterpGRD=' '
-waveuoutpGRD='gwes_30m'
-MESH_WAV='mesh.gwes_30m.nc'
+
+waveGRD='mx025'
+waveinterpGRD='reg025'
+waveuoutpGRD='mx025'
+MESH_WAV='mesh.mx025.nc'
 
 waveesmfGRD=' '
-wavepostGRD='gwes_30m'
+wavepostGRD=' '
 waveGRDN="1"
 waveGRDG="10"
 USE_WAV_RMP="NO"

diff --git a/parm/config/config.fcst b/parm/config/config.fcst
@@ -161,6 +161,11 @@ export isatmedmf=1
 tbf=""
 if [[ "$satmedmf" = ".true." ]]; then tbf="_satmedmf" ; fi
 
+#Convection schemes
+export progsigma=".true."
+tbp=""
+if [ "$progsigma" = ".true." ]; then tbp="_progsigma" ; fi
+
 # Radiation options
 export IAER=1011    ; #spectral band mapping method for aerosol optical properties
 export iovr_lw=3    ; #de-correlation length cloud overlap method (Barker, 2008)
@@ -188,17 +193,17 @@ export random_clds=".true."
 
 if [[ "$imp_physics" -eq 99 ]]; then # ZhaoCarr
     export ncld=1
-    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_zhaocarr${tbf}"
+    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_zhaocarr${tbf}${tbp}"
     export nwat=2
 
 elif [[ "$imp_physics" -eq 6 ]]; then # WSM6
     export ncld=2
-    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_wsm6${tbf}"
+    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_wsm6${tbf}${tbp}"
     export nwat=6
 
 elif [[ "$imp_physics" -eq 8 ]]; then # Thompson
     export ncld=2
-    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_thompson_noaero_tke"
+    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_thompson_noaero_tke${tbp}"
     export nwat=6
 
     export cal_pre=".false."
@@ -220,7 +225,7 @@ elif [[ "$imp_physics" -eq 8 ]]; then # Thompson
     export d4_bg=0.12
 elif [[ "$imp_physics" -eq 11 ]]; then # GFDL
     export ncld=5
-    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_gfdl${tbf}"
+    export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_gfdl${tbf}${tbp}"
     export nwat=6
     export dnats=1
     export cal_pre=".false."

diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
@@ -156,8 +156,11 @@ case ${case_in} in
         ;;
 esac
 
-if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi
-if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi
+#While in theory this is likely the best way to go, in practice partiularly for C768 runs will fail
+#when using npe_node_max.  Additional write groups also did not get around errors. Should likely 
+#be revisted at another time.
+#if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi
+#if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi
 
 # Calculate chunksize based on resolution
 export RESTILE=$(echo ${case_in} |cut -c2-)

diff --git a/parm/config/config.wavepostsbs b/parm/config/config.wavepostsbs
@@ -17,7 +17,7 @@ export DOIBP_WAV='NO' # Input boundary points
 export DOFLD_WAV='YES' # Field data
 export DOPNT_WAV='YES' # Station data
 export DOGRB_WAV='YES' # Create grib2 files
-export DOGRI_WAV='NO' # Create interpolated grids
+export DOGRI_WAV='YES' # Create interpolated grids
 export DOSPC_WAV='YES' # Spectral post
 export DOBLL_WAV='YES' # Bulletin post
 

diff --git a/parm/mom6/MOM_input_template_025 b/parm/mom6/MOM_input_template_025
@@ -6,7 +6,6 @@
 ! This MOM_input file typically contains only the non-default values that are needed to reproduce this example.
 ! A full list of parameters for this example can be found in the corresponding MOM_parameter_doc.all file
 ! which is generated by the model at run-time.
-
 ! === module MOM_domains ===
 TRIPOLAR_N = True               !   [Boolean] default = False
                                 ! Use tripolar connectivity at the northern edge of the domain.  With
@@ -202,6 +201,7 @@ DTFREEZE_DP = -7.75E-08         !   [deg C Pa-1] default = 0.0
 PARALLEL_RESTARTFILES = True    !   [Boolean] default = False
                                 ! If true, each processor writes its own restart file, otherwise a single
                                 ! restart file is generated
+STORE_CORIOLIS_ACCEL = False
 
 ! === module MOM_tracer_flow_control ===
 USE_IDEAL_AGE_TRACER = False    !   [Boolean] default = False
@@ -510,6 +510,9 @@ USE_LAND_MASK_FOR_HVISC = False !   [Boolean] default = False
 HMIX_FIXED = 0.5                !   [m]
                                 ! The prescribed depth over which the near-surface viscosity and diffusivity are
                                 ! elevated when the bulk mixed layer is not used.
+KVML = 1.0E-04                  !   [m2 s-1] default = 1.0E-04
+                                ! The kinematic viscosity in the mixed layer.  A typical value is ~1e-2 m2 s-1.
+                                ! KVML is not used if BULKMIXEDLAYER is true.  The default is set by KV.
 MAXVEL = 6.0                    !   [m s-1] default = 3.0E+08
                                 ! The maximum velocity allowed before the velocity components are truncated.
 

diff --git a/parm/mom6/MOM_input_template_050 b/parm/mom6/MOM_input_template_050
@@ -6,7 +6,6 @@
 ! This MOM_input file typically contains only the non-default values that are needed to reproduce this example.
 ! A full list of parameters for this example can be found in the corresponding MOM_parameter_doc.all file
 ! which is generated by the model at run-time.
-
 ! === module MOM_domains ===
 TRIPOLAR_N = True               !   [Boolean] default = False
                                 ! Use tripolar connectivity at the northern edge of the domain.  With
@@ -200,6 +199,7 @@ DTFREEZE_DP = -7.75E-08         !   [deg C Pa-1] default = 0.0
 PARALLEL_RESTARTFILES = True    !   [Boolean] default = False
                                 ! If true, each processor writes its own restart file, otherwise a single
                                 ! restart file is generated
+STORE_CORIOLIS_ACCEL = False
 
 ! === module MOM_tracer_flow_control ===
 USE_IDEAL_AGE_TRACER = False    !   [Boolean] default = False
@@ -540,6 +540,9 @@ USE_LAND_MASK_FOR_HVISC = False !   [Boolean] default = False
 HMIX_FIXED = 0.5                !   [m]
                                 ! The prescribed depth over which the near-surface viscosity and diffusivity are
                                 ! elevated when the bulk mixed layer is not used.
+KVML = 1.0E-04                  !   [m2 s-1] default = 1.0E-04
+                                ! The kinematic viscosity in the mixed layer.  A typical value is ~1e-2 m2 s-1.
+                                ! KVML is not used if BULKMIXEDLAYER is true.  The default is set by KV.
 MAXVEL = 6.0                    !   [m s-1] default = 3.0E+08
                                 ! The maximum velocity allowed before the velocity components are truncated.
 

diff --git a/parm/mom6/MOM_input_template_100 b/parm/mom6/MOM_input_template_100
@@ -1,5 +1,4 @@
 ! This file was written by the model and records all non-layout or debugging parameters used at run-time.
-
 ! === module MOM ===
 
 ! === module MOM_unit_scaling ===
@@ -211,6 +210,7 @@ TFREEZE_FORM = "MILLERO_78"     ! default = "LINEAR"
 PARALLEL_RESTARTFILES = True    !   [Boolean] default = False
                                 ! If true, each processor writes its own restart file, otherwise a single
                                 ! restart file is generated
+STORE_CORIOLIS_ACCEL = False
 
 ! === module MOM_tracer_flow_control ===
 USE_IDEAL_AGE_TRACER = False     !   [Boolean] default = False
@@ -531,6 +531,9 @@ USE_KH_BG_2D = True             !   [Boolean] default = False
 HMIX_FIXED = 0.5                !   [m]
                                 ! The prescribed depth over which the near-surface viscosity and diffusivity are
                                 ! elevated when the bulk mixed layer is not used.
+KVML = 1.0E-04                  !   [m2 s-1] default = 1.0E-04
+                                ! The kinematic viscosity in the mixed layer.  A typical value is ~1e-2 m2 s-1.
+                                ! KVML is not used if BULKMIXEDLAYER is true.  The default is set by KV.
 MAXVEL = 6.0                    !   [m s-1] default = 3.0E+08
                                 ! The maximum velocity allowed before the velocity components are truncated.
 

diff --git a/parm/mom6/MOM_input_template_500 b/parm/mom6/MOM_input_template_500
@@ -1,5 +1,4 @@
 ! This file was written by the model and records the non-default parameters used at run-time.
-
 ! === module MOM ===
 
 ! === module MOM_unit_scaling ===
@@ -21,7 +20,7 @@ DT_THERM = @[DT_THERM_MOM6]     !   [s] default = 1800.0
                                 ! an integer multiple of DT and less than the forcing or coupling time-step,
                                 ! unless THERMO_SPANS_COUPLING is true, in which case DT_THERM can be an integer
                                 ! multiple of the coupling timestep.  By default DT_THERM is set to DT.
-THERMO_SPANS_COUPLING = False    !   [Boolean] default = False
+THERMO_SPANS_COUPLING = @[MOM6_THERMO_SPAN]    !   [Boolean] default = False
                                 ! If true, the MOM will take thermodynamic and tracer timesteps that can be
                                 ! longer than the coupling timestep. The actual thermodynamic timestep that is
                                 ! used in this case is the largest integer multiple of the coupling timestep
@@ -43,10 +42,10 @@ BOUND_SALINITY = True           !   [Boolean] default = False
 TRIPOLAR_N = True               !   [Boolean] default = False
                                 ! Use tripolar connectivity at the northern edge of the domain.  With
                                 ! TRIPOLAR_N, NIGLOBAL must be even.
-NIGLOBAL =  @[NX_GLB]           !
+NIGLOBAL = @[NX_GLB]            !
                                 ! The total number of thickness grid points in the x-direction in the physical
                                 ! domain. With STATIC_MEMORY_ this is set in MOM_memory.h at compile time.
-NJGLOBAL =  @[NY_GLB]           !
+NJGLOBAL = @[NY_GLB]            !
                                 ! The total number of thickness grid points in the y-direction in the physical
                                 ! domain. With STATIC_MEMORY_ this is set in MOM_memory.h at compile time.
 
@@ -133,6 +132,7 @@ TFREEZE_FORM = "MILLERO_78"     ! default = "LINEAR"
 
 ! === module MOM_restart ===
 RESTART_CHECKSUMS_REQUIRED = False
+STORE_CORIOLIS_ACCEL = False
 ! === module MOM_tracer_flow_control ===
 
 ! === module MOM_coord_initialization ===
@@ -342,6 +342,9 @@ USE_LAND_MASK_FOR_HVISC = True  !   [Boolean] default = False
 HMIX_FIXED = 0.5                !   [m]
                                 ! The prescribed depth over which the near-surface viscosity and diffusivity are
                                 ! elevated when the bulk mixed layer is not used.
+KVML = 1.0E-04                  !   [m2 s-1] default = 1.0E-04
+                                ! The kinematic viscosity in the mixed layer.  A typical value is ~1e-2 m2 s-1.
+                                ! KVML is not used if BULKMIXEDLAYER is true.  The default is set by KV.
 MAXVEL = 6.0                    !   [m s-1] default = 3.0E+08
                                 ! The maximum velocity allowed before the velocity components are truncated.
 
@@ -451,10 +454,10 @@ MAX_RINO_IT = 25                !   [nondim] default = 50
 EPBL_USTAR_MIN = 1.45842E-18    !   [m s-1]
                                 ! The (tiny) minimum friction velocity used within the ePBL code, derived from
                                 ! OMEGA and ANGSTROM..
-USE_LA_LI2016 = True !   [nondim] default = False
+USE_LA_LI2016 = @[MOM6_USE_LI2016] !   [nondim] default = False
                                 ! A logical to use the Li et al. 2016 (submitted) formula to determine the
                                 ! Langmuir number.
-USE_WAVES = False   !   [Boolean] default = False
+USE_WAVES = @[MOM6_USE_WAVES]   !   [Boolean] default = False
                                 ! If true, enables surface wave modules.
 
 ! === module MOM_regularize_layers ===

diff --git a/parm/parm_fv3diag/field_table_gfdl_progsigma b/parm/parm_fv3diag/field_table_gfdl_progsigma
@@ -0,0 +1,42 @@
+# added by FRE: sphum must be present in atmos
+# specific humidity for moist runs
+ "TRACER", "atmos_mod", "sphum"
+           "longname",     "specific humidity"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic cloud water mixing ratio
+ "TRACER", "atmos_mod", "liq_wat"
+           "longname",     "cloud water mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+ "TRACER", "atmos_mod", "rainwat"
+           "longname",     "rain mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+ "TRACER", "atmos_mod", "ice_wat"
+           "longname",     "cloud ice mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+ "TRACER", "atmos_mod", "snowwat"
+           "longname",     "snow mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+ "TRACER", "atmos_mod", "graupel"
+           "longname",     "graupel mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic ozone mixing ratio tracer
+ "TRACER", "atmos_mod", "o3mr"
+           "longname",     "ozone mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognotsitc sigmab tracer
+ "TRACER", "atmos_mod", "sigmab"
+           "longname",     "sigma fraction"
+           "units",        "fraction"
+       "profile_type", "fixed", "surface_value=0.0" /
+# non-prognostic cloud amount
+ "TRACER", "atmos_mod", "cld_amt"
+           "longname",     "cloud amount"
+           "units",        "1"
+       "profile_type", "fixed", "surface_value=1.e30" /