[WIP] Decomp init study branch

.gitmodules

@@ -99,8 +99,11 @@ fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git
 
 [submodule "share"]
 path = share
-url = https://github.com/ESCOMP/CESM_share
-fxtag = share1.1.9
+#url = https://github.com/ESCOMP/CESM_share
+url = https://github.com/ekluzek/CESM_share
+#fxtag = share1.1.9
+#fxtag = add_jdennis_procstatus_module
+fxtag = 9973692556da54f9562935be43c1d43b0607d24b
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share

bld/namelist_files/namelist_definition_ctsm.xml

@@ -1242,12 +1242,23 @@ Whether to use subgrid fluxes for snow
 Whether snow on the vegetation canopy affects the radiation/albedo calculations
 </entry>
 
+<entry id="for_testing_exit_after_self_tests" type="logical" category="default_settings"
+       group="clm_inparm" >
+Whether to exit early after the initialization self tests are run. This is typically only used in automated tests.
+</entry>
+
 <entry id="for_testing_run_ncdiopio_tests" type="logical" category="default_settings"
        group="clm_inparm" >
 Whether to run some tests of ncdio_pio as part of the model run. This is
 typically only used in automated tests.
 </entry>
 
+<entry id="for_testing_run_decomp_init_tests" type="logical" category="default_settings"
+       group="clm_inparm" >
+Whether to run some tests of decompInit (to get the gridcell to MPI task decomposition) as part of the model run. This is
+typically only used in automated tests.
+</entry>
+
 <entry id="for_testing_use_second_grain_pool" type="logical" category="default_settings"
        group="clm_inparm" >
 If true, allocate memory for and use a second crop grain pool. This is

cime_config/config_pes.xml

@@ -2092,6 +2092,44 @@
 	</rootpe>
       </pes>
     </mach>
+  </grid>
+   <grid name="l%mpasa3p75">
+    <mach name="derecho">
+      <pes pesize="any" compset="any">
+	<comment>none</comment>
+	<ntasks>
+	  <ntasks_atm>-1</ntasks_atm>
+	  <ntasks_lnd>-80</ntasks_lnd>
+	  <ntasks_rof>-80</ntasks_rof>
+	  <ntasks_ice>-80</ntasks_ice>
+	  <ntasks_ocn>-80</ntasks_ocn>
+	  <ntasks_glc>-80</ntasks_glc>
+	  <ntasks_wav>-80</ntasks_wav>
+	  <ntasks_cpl>-80</ntasks_cpl>
+	  <ntasks_lnd>-80</ntasks_lnd>
+	</ntasks>
+	<nthrds>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_glc>1</nthrds_glc>
+	  <nthrds_wav>1</nthrds_wav>
+	  <nthrds_cpl>1</nthrds_cpl>
+	</nthrds>
+	<rootpe>
+	  <rootpe_atm>0</rootpe_atm>
+	  <rootpe_lnd>-1</rootpe_lnd>
+	  <rootpe_rof>-1</rootpe_rof>
+	  <rootpe_ice>-1</rootpe_ice>
+	  <rootpe_ocn>-1</rootpe_ocn>
+	  <rootpe_glc>-1</rootpe_glc>
+	  <rootpe_wav>-1</rootpe_wav>
+	  <rootpe_cpl>-1</rootpe_cpl>
+	</rootpe>
+      </pes>
+    </mach>
   </grid>
   <grid name="l%0.125nldas2">
     <mach name="any">

cime_config/testdefs/ExpectedTestFails.xml

@@ -363,11 +363,4 @@
 
   <!-- decomp_init test list-->
 
-  <test name="SMS_Ln1_PL.mpasa15_mpasa15.I2000Clm45Sp.derecho_intel.clm-run_self_tests">
-    <phase name="RUN">
-      <status>FAIL</status>
-      <issue>#3316</issue>
-    </phase>
-  </test>
-
 </expectedFails>

cime_config/testdefs/testlist_clm.xml

@@ -14,6 +14,7 @@
         matrixcn: Tests exercising the matrix-CN capability
         aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy.
         decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization
+        uhr_decomp_init: Initialization tests at Ultra High Resolution - specifically for examining the PE layout decomposition initialization
 -->
 <testlist version="2.0">
   <test name="ERI_D_Ld9" grid="f10_f10_mg37" compset="I1850Clm60Bgc" testmods="clm/default">
@@ -4209,6 +4210,15 @@
       <option name="comment">Initialization test for mpasa15 with "Large" layout</option>
     </options>
   </test>
+  <test name="SMS_Ln1" grid="mpasa3p75_mpasa3p75_mt13" compset="I2000Clm45Sp" testmods="clm-run_self_tests--clm-mpasa3p75">
+    <machines>
+      <machine name="derecho" compiler="intel" category="uhr_decomp_init"/>
+    </machines>
+    <options>
+      <option name="wallclock">0:15:00</option>
+      <option name="comment">Initialization test for mpasa3p75 default layout</option>
+    </options>
+  </test>
 
   <test name="LGRAIN2_Ly1_P128x1" grid="f10_f10_mg37" compset="I1850Clm50BgcCrop" testmods="clm/ciso--clm/cropMonthOutput">
     <machines>

cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm

@@ -0,0 +1,6 @@
+! Settings currently required to run at the mpasa3p75 grid
+! urbantv files at that resolution and use a redistribution mapping
+
+stream_fldfilename_urbantv = '/glade/derecho/scratch/bdobbins/ko/tbuildmax.nc'
+stream_meshfile_urbantv = '/glade/derecho/scratch/bdobbins/ko/mesh.nc'
+urbantvmapalgo = 'redist'

cime_config/testdefs/testmods_dirs/clm/run_self_tests/README

@@ -1,5 +1,10 @@
-The purpose of this testmod directory is to trigger the runtime
-self-tests. This runs a suite of unit/integration tests.
+The purpose of this testmod directory is to trigger runtime
+initialization self-tests. This runs a set of unit/integration tests
+that apply at initialization.
 
 We use cold start so that we can get through initialization faster,
 since how we initialize the model is unimportant for these self-tests.
+We also exit as early as possible to minimize the time spent.
+
+There are other self_tests that need to be exercised in the model time stepping
+and are done outside of these.

cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands

@@ -4,12 +4,25 @@
 # We use this testmod in a _Ln1 test; this requires forcing the ROF coupling frequency to same frequency as DATM
 ./xmlchange ROF_NCPL='$ATM_NCPL'
 
+# Turn off ROF model when used with compsets that have them
+./xmlchange RTM_MODE='NULL'
+
 # Turn MEGAN off to run faster
 ./xmlchange CLM_BLDNML_OPTS='--no-megan' --append
 
 # Use fast structure and NWP configuration for speed
 ./xmlchange CLM_STRUCTURE="fast"
 ./xmlchange CLM_CONFIGURATION="nwp"
 
-# Turn cpl history off
-./xmlchange HIST_OPTION="never"
+# Restarts aren't allowed for these tests, and turn off CPL history
+# First change in env_test.xml, then in the standard one so it won't complain there
+./xmlchange --force REST_OPTION="never" --file env_test.xml
+./xmlchange --force HIST_OPTION="never" --file env_test.xml
+./xmlchange REST_OPTION="never"
+./xmlchange HIST_OPTION="never"
+
+# Timer settings
+./xmlchange TIMER_DETAIL="2"
+./xmlchange SAVE_TIMING="TRUE"
+./xmlchange CHECK_TIMING="TRUE"
+./xmlchange ESMF_PROFILING_LEVEL="10"

cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm

@@ -1 +1,9 @@
-for_testing_run_ncdiopio_tests = .true.
+for_testing_run_ncdiopio_tests = .false.
+for_testing_run_decomp_init_tests = .true.
+for_testing_exit_after_self_tests = .true.
+
+! Turn off history, restarts, and output
+hist_empty_htapes = .true.
+use_noio = .true.
+urban_hac = 'OFF'
+toosmall_urban =  98.0d00  ! Minimize urban in gridcells

src/cpl/nuopc/lnd_comp_nuopc.F90

@@ -39,6 +39,7 @@ module lnd_comp_nuopc
   use clm_varctl             , only : single_column, clm_varctl_set, iulog
   use clm_varctl             , only : nsrStartup, nsrContinue, nsrBranch
   use clm_varctl             , only : FL => fname_len
+  use clm_varctl             , only : for_testing_exit_after_self_tests
   use clm_time_manager       , only : set_timemgr_init, advance_timestep
   use clm_time_manager       , only : update_rad_dtime
   use clm_time_manager       , only : get_nstep, get_step_size
@@ -80,6 +81,7 @@ module lnd_comp_nuopc
 
   logical                :: glc_present
   logical                :: rof_prognostic
+  logical                :: atm_present
   logical                :: atm_prognostic
   integer, parameter     :: dbug = 0
   character(*),parameter :: modName =  "(lnd_comp_nuopc)"
@@ -284,6 +286,11 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     else
        atm_prognostic = .true.
     end if
+    if (trim(atm_model) == 'satm') then
+       atm_present = .false.
+    else
+       atm_present = .true.
+    end if
     call NUOPC_CompAttributeGet(gcomp, name='GLC_model', value=glc_model, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     if (trim(glc_model) == 'sglc') then
@@ -310,6 +317,9 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
        write(iulog,'(a   )')' rof component                 = '//trim(rof_model)
        write(iulog,'(a   )')' glc component                 = '//trim(glc_model)
        write(iulog,'(a,L2)')' atm_prognostic                = ',atm_prognostic
+       if (.not. atm_present) then
+          write(iulog,'(a,L2)')' atm_present                  = ',atm_present
+       end if
        write(iulog,'(a,L2)')' rof_prognostic                = ',rof_prognostic
        write(iulog,'(a,L2)')' glc_present                   = ',glc_present
        if (glc_present) then
@@ -328,7 +338,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     call control_setNL("lnd_in"//trim(inst_suffix))
 
 
-    call advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, atm_prognostic, rc)
+    call advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, &
+                          atm_prognostic, atm_present, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     !----------------------------------------------------------------------------
@@ -492,6 +503,12 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
     else
        single_column = .false.
     end if
+    if ( for_testing_exit_after_self_tests) then
+       ! *******************
+       ! *** RETURN HERE ***
+       ! *******************
+       RETURN
+    end if
 
     !----------------------------------------------------------------------------
     ! Reset shr logging to my log file
@@ -771,6 +788,9 @@ subroutine ModelAdvance(gcomp, rc)
     if (single_column .and. .not. scol_valid) then
        RETURN
     end if
+    if (for_testing_exit_after_self_tests) then
+       RETURN
+    end if
 
     !$  call omp_set_num_threads(nthrds)
 
@@ -1002,6 +1022,7 @@ subroutine ModelSetRunClock(gcomp, rc)
     rc = ESMF_SUCCESS
     call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO)
     if (.not. scol_valid) return
+    if (for_testing_exit_after_self_tests) return
 
     ! query the Component for its clocks
     call NUOPC_ModelGet(gcomp, driverClock=dclock, modelClock=mclock, rc=rc)
@@ -1285,6 +1306,7 @@ subroutine clm_orbital_update(clock, logunit,  mastertask, eccen, obliqr, lambm0
   end subroutine clm_orbital_update
 
   subroutine CheckImport(gcomp, rc)
+    use clm_varctl, only : for_testing_exit_after_self_tests
     type(ESMF_GridComp) :: gcomp
     integer, intent(out) :: rc
     character(len=*) , parameter :: subname = "("//__FILE__//":CheckImport)"
@@ -1313,6 +1335,9 @@ subroutine CheckImport(gcomp, rc)
     if (single_column .and. .not. scol_valid) then
        RETURN
     end if
+    if (for_testing_exit_after_self_tests) then
+       RETURN
+    end if
     ! The remander of this should be equivalent to the NUOPC internal routine
     ! from NUOPC_ModeBase.F90
 

src/cpl/nuopc/lnd_import_export.F90

@@ -156,7 +156,8 @@ module lnd_import_export
 contains
 !===============================================================================
 
-  subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, atm_prognostic, rc)
+  subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, &
+                              atm_prognostic, atm_present, rc)
 
     use shr_carma_mod     , only : shr_carma_readnl
     use shr_ndep_mod      , only : shr_ndep_readnl
@@ -173,6 +174,7 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r
     logical          , intent(in)  :: cism_evolve
     logical          , intent(in)  :: rof_prognostic
     logical          , intent(in)  :: atm_prognostic
+    logical          , intent(in)  :: atm_present
     integer          , intent(out) :: rc
 
     ! local variables
@@ -210,7 +212,9 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r
 
     ! Need to determine if there is no land for single column before the advertise call is done
 
-    if (atm_prognostic .or. force_send_to_atm) then
+    if (.not. atm_present)then
+       send_to_atm = .false.
+    else if (atm_prognostic .or. force_send_to_atm) then
        send_to_atm = .true.
     else
        send_to_atm = .false.
@@ -253,12 +257,11 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r
     if (shr_megan_mechcomps_n .ne. megan_nflds) call shr_sys_abort('ERROR: megan field count mismatch')
 
     ! CARMA volumetric soil water from land
-    call shr_carma_readnl('drv_flds_in', carma_fields)
 
     ! export to atm
     call fldlist_add(fldsFrLnd_num, fldsFrlnd, trim(flds_scalar_name))
-    call fldlist_add(fldsFrLnd_num, fldsFrlnd, 'Sl_lfrin')
     if (send_to_atm) then
+       call fldlist_add(fldsFrLnd_num, fldsFrlnd, 'Sl_lfrin')
        call fldlist_add(fldsFrLnd_num, fldsFrlnd, Sl_t          )
        call fldlist_add(fldsFrLnd_num, fldsFrlnd, Sl_tref       )
        call fldlist_add(fldsFrLnd_num, fldsFrlnd, Sl_qref       )
@@ -339,6 +342,9 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r
 
     call fldlist_add(fldsToLnd_num, fldsToLnd, trim(flds_scalar_name))
 
+    !!!!!!!!!!!!!!!!!!!!!!!!!!! new if section !!!!!!!!!!!!!!!!!!!!!!!!!!
+    if ( atm_present ) then
+
     ! from atm
     call fldlist_add(fldsToLnd_num, fldsToLnd, Sa_z         )
     call fldlist_add(fldsToLnd_num, fldsToLnd, Sa_topo      )
@@ -389,6 +395,9 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r
        call fldlist_add(fldsToLnd_num, fldsToLnd, Sa_co2diag)
     end if
 
+    end if ! atm_present
+    !!!!!!!!!!!!!!!!!!!!!!!!!!! new if section !!!!!!!!!!!!!!!!!!!!!!!!!!
+
     if (rof_prognostic) then
        ! from river
        call fldlist_add(fldsToLnd_num, fldsToLnd, Flrr_flood   )
@@ -773,14 +782,14 @@ subroutine export_fields( gcomp, bounds, glc_present, rof_prognostic, &
     ! output to mediator
     ! -----------------------
 
+    if (send_to_atm) then
     call state_setexport_1d(exportState, Sl_lfrin, ldomain%frac(begg:), init_spval=.false., rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     ! -----------------------
     ! output to atm
     ! -----------------------
 
-    if (send_to_atm) then
        call state_setexport_1d(exportState, Sl_t      , lnd2atm_inst%t_rad_grc(begg:), &
             init_spval=.true., rc=rc)
        if (ChkErr(rc,__LINE__,u_FILE_u)) return

src/cpl/share_esmf/UrbanTimeVarType.F90

@@ -16,6 +16,7 @@ module UrbanTimeVarType
   use clm_varcon      , only : spval
   use LandunitType    , only : lun
   use GridcellType    , only : grc
+  use perf_mod        , only : t_startf, t_stopf
   !
   implicit none
   private
@@ -143,6 +144,8 @@ subroutine urbantv_init(this, bounds, NLFilename)
          stream_meshfile_urbantv, &
          urbantv_tintalgo
 
+    call t_startf("urbantv_init")
+
     ! Default values for namelist
     stream_year_first_urbantv  = 1       ! first year in stream to use
     stream_year_last_urbantv   = 1       ! last  year in stream to use
@@ -195,6 +198,7 @@ subroutine urbantv_init(this, bounds, NLFilename)
     endif
 
     ! Initialize the cdeps data type this%sdat_urbantv
+    call t_startf("str_data_init")
     call shr_strdata_init_from_inline(this%sdat_urbantv,             &
          my_task             = iam,                                  &
          logunit             = iulog,                                &
@@ -219,6 +223,9 @@ subroutine urbantv_init(this, bounds, NLFilename)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then
        call ESMF_Finalize(endflag=ESMF_END_ABORT)
     end if
+    call t_stopf("str_data_init")
+
+    call t_stopf("urbantv_init")
 
   end subroutine urbantv_init