add clarifying comments to the fates seed dispersal code

bld/namelist_files/namelist_definition_ctsm.xml

@@ -642,6 +642,10 @@ Switch deciding which nutrient model to use in FATES.
         group="clm_inparm" valid_values="0,1,2,3">
 Switch defining the cadence at which seeds are dispersed across
 gridcells.  Setting the switch value to zero turns off dispersal.
+Setting the switch to 1, 2, or 3 sets the dispersal cadence to
+daily, monthly or yearly.  The daily cadence is primarily
+recommended for test and debug only. Note that turning this
+feature on will result in more memory usage.
 (Only relevant if FATES is on)
 </entry>
 

cime_config/testdefs/testlist_clm.xml

@@ -1841,6 +1841,7 @@
     </machines>
     <options>
       <option name="wallclock">00:40:00</option>
+      <option name="comment"  >This test should be convered to an ERP test once the PEM version of this test is passing COMPARE_base_modpes.</option>
     </options>
   </test>
   <test name="PEM_D_Ld15" grid="5x5_amazon" compset="I2000Clm50FatesRs" testmods="clm/FatesColdSeedDisp">
@@ -1849,6 +1850,7 @@
     </machines>
     <options>
       <option name="wallclock">00:40:00</option>
+      <option name="comment"  >This checks that the FATES seed dispersal MPI communication is consistent when the PE layout changes.</option>
     </options>
   </test>
   <test name="ERS_D_Ld15" grid="f45_f45_mg37" compset="I2000Clm50FatesRs" testmods="clm/FatesColdTreeDamage">

cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/README

@@ -9,7 +9,7 @@ fates_seed_dispersal_pdf_shape
 
 Given that the default fates parameter file has the above variables as unset,
 a custom fates parameter file must be supplied to appropriately test this mode.
-This testmod itself addresses CTSM issue #2151.
+This testmod itself addresses CTSM issue 2151: https://github.com/ESCOMP/CTSM/issues/2151
 
 The max_dist value will impact the size of the 'neighborhood' of gridcells
 that fates will attempt to distribute seeds to.  To limit the neighborhood to

src/main/clm_driver.F90

@@ -1271,6 +1271,8 @@ subroutine clm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate, ro
 
 
     ! Pass fates seed dispersal information to all nodes
+    ! Note that WrapSeedGlobal calls an MPI collective routine and as such
+    ! WrapSeedGlobal should be called outside of OMP threaded loop regions
     if (fates_seeddisp_cadence /= fates_dispersal_cadence_none) then
        if (use_fates .and. is_beg_curr_day()) call clm_fates%WrapSeedGlobal()
     end if

src/main/controlMod.F90

@@ -1112,6 +1112,7 @@ subroutine control_print ()
        write(iulog, *) '    use_fates_nocomp = ', use_fates_nocomp
        write(iulog, *) '    use_fates_sp = ', use_fates_sp
        write(iulog, *) '    fates_seeddisp_cadence = ', fates_seeddisp_cadence
+       write(iulog, *) '    fates_seeddisp_cadence: 0, 1, 2, 3 => off, daily, monthly, or yearly dispersal'
        write(iulog, *) '    fates_inventory_ctrl_filename = ',fates_inventory_ctrl_filename
     end if
   end subroutine control_print

src/utils/clmfates_interfaceMod.F90

@@ -2682,11 +2682,16 @@ subroutine WrapSeedGlobal(this,is_restart_flag)
       this%fates_seed%incoming_global(:,:) = 0._r8
       this%fates_seed%outgoing_global(:,:) = 0._r8
 
-      ! Distribute outgoing seed data from all nodes to all nodes
+      ! Distribute outgoing seed data across all MPI tasks
+      ! This method of grid cell communications is inefficient in that it creates global information
+      ! across all processes.  A future update should communicate to the minimum set of neighbors.
       call MPI_Allgatherv(this%fates_seed%outgoing_local, procinfo%ncells*numpft_fates, MPI_REAL8, &
                           this%fates_seed%outgoing_global, this%fates_seed%ncells_array*numpft_fates, this%fates_seed%begg_array*numpft_fates, &
                           MPI_REAL8, mpicom, ier)
-      write(iulog,*) 'WSG: MPI_Allgatherv ier: ', ier
+      if (ier /= 0) then
+         call endrun(msg='clmfates interface error: MPI_Allgatherv failed'//&
+              errMsg(sourcefile, __LINE__))
+      end if
 
       ! zero outgoing local for all gridcells outside threaded region now that we've passed them out
       this%fates_seed%outgoing_local(:,:) = 0._r8
@@ -2716,7 +2721,9 @@ end subroutine WrapSeedGlobal
 
  subroutine wrap_seed_dispersal(this,bounds_clump)
 
-    ! This subroutine pass seed_id_global to bc_in and reset seed_out
+    ! This subroutine passes the globally dispersed seed via WrapSeedGlobal, incoming_global
+    ! to the fates local process seed_in site object.  It also resets the fates seed_out
+    ! in preparation for fates to update the seeds being dispersed out.
 
     ! Arguments
     class(hlm_fates_interface_type), intent(inout) :: this