Moving Nest Code Restructure and Port to fms2_io (#227)

* Performance optimization of moving nest. * Moving nest performance optimization stage 2. * Update atmos_model_nml in driver/fvGFS/atmosphere.F90 so that it is consistent with that in FV3/atmos_model.F90. * Adding upoff as a namelist parameter * Moving nest code cleanup: modularization phase. * Ported from fms_io to fms2_io. Also removed debug print statements. * Restructuring moving_nest code from atmos_cubed_sphere to FV3 level. * Added namelist flag fv_timings to enable detailed performance timings; defaults to false. Co-authored-by: Bin Liu <Bin.Liu@noaa.gov> Co-authored-by: Biju Thomas <biju.thomas@noaa.gov>
NOAA-GFDL · Dec 14, 2022 · a839395 · a839395
1 parent 2cdf37b
commit a839395
Show file tree

Hide file tree

Showing 15 changed files with 23 additions and 12,303 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -58,15 +58,6 @@ if(NOT FMS_FOUND)
   add_library(fms ALIAS FMS::fms_${kind_lower})
 endif()
 
-list(APPEND moving_srcs
-  moving_nest/bounding_box.F90
-  moving_nest/fv_tracker.F90
-  moving_nest/fv_moving_nest.F90
-  moving_nest/fv_moving_nest_main.F90
-  moving_nest/fv_moving_nest_physics.F90
-  moving_nest/fv_moving_nest_types.F90
-  moving_nest/fv_moving_nest_utils.F90)
-
 list(APPEND model_srcs
   model/a2b_edge.F90
   model/multi_gases.F90
@@ -121,7 +112,6 @@ list(APPEND driver_srcs
   driver/fvGFS/atmosphere.F90)
 
 list(APPEND fv3_srcs ${model_srcs}
-                     ${moving_srcs}
                      ${tools_srcs})
 
 list(APPEND fv3_defs SPMD

diff --git a/driver/fvGFS/atmosphere.F90 b/driver/fvGFS/atmosphere.F90
@@ -196,11 +196,6 @@ module atmosphere_mod
 use fv_nesting_mod,     only: twoway_nesting
 use boundary_mod,       only: fill_nested_grid
 use fv_diagnostics_mod, only: fv_diag_init, fv_diag, fv_time, prt_maxmin, prt_height
-#ifdef MOVING_NEST
-use fv_tracker_mod,     only: fv_diag_tracker, allocate_tracker
-use fv_tracker_mod,     only: fv_tracker_init, fv_tracker_center, fv_tracker_post_move
-use fv_moving_nest_types_mod, only: Moving_nest
-#endif
 use fv_nggps_diags_mod, only: fv_nggps_diag_init, fv_nggps_diag, fv_nggps_tavg
 use fv_restart_mod,     only: fv_restart, fv_write_restart
 use fv_timing_mod,      only: timing_on, timing_off
@@ -264,7 +259,6 @@ module atmosphere_mod
   integer :: nq                       !  number of transported tracers
   integer :: sec, seconds, days
   integer :: id_dynam, id_fv_diag, id_subgridz
-  integer :: id_fv_tracker
 
   logical :: cold_start = .false.     !  used in initial condition
 
@@ -329,11 +323,9 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
    logical :: debug        = .false.
    logical :: sync         = .false.
    logical :: ignore_rst_cksum = .false.
-   integer, parameter     :: maxhr = 4096
-   real, dimension(maxhr) :: fdiag = 0.
-   real                   :: fhmax=384.0, fhmaxhf=120.0, fhout=3.0, fhouthf=1.0,avg_max_length=3600.
-   namelist /atmos_model_nml/ blocksize, chksum_debug, dycore_only, debug, sync, fdiag, fhmax, fhmaxhf, fhout, fhouthf, ccpp_suite, &
-                            & avg_max_length, ignore_rst_cksum
+   real    :: avg_max_length = 3600.
+   namelist /atmos_model_nml/ blocksize, chksum_debug, dycore_only, debug, sync, ccpp_suite, avg_max_length, &
+                              ignore_rst_cksum
    ! *DH 20210326
 
    !For regional
@@ -359,12 +351,6 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
 
    call fv_control_init( Atm, dt_atmos, mygrid, grids_on_this_pe, p_split )  ! allocates Atm components; sets mygrid
 
-   ! TODO move this higher into atmos_model.F90 for better modularization
-#ifdef MOVING_NEST
-   call fv_tracker_init(size(Atm))
-   if (mygrid .eq. 2) call allocate_tracker(mygrid, Atm(mygrid)%bd%isc, Atm(mygrid)%bd%iec, Atm(mygrid)%bd%jsc, Atm(mygrid)%bd%jec)
-#endif
-
    Atm(mygrid)%Time_init = Time_init
 
    if(Atm(mygrid)%flagstruct%warm_start) then
@@ -499,9 +485,6 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
    id_subgridz  = mpp_clock_id ('FV subgrid_z',flags = clock_flag_default, grain=CLOCK_SUBCOMPONENT )
    id_fv_diag   = mpp_clock_id ('FV Diag',     flags = clock_flag_default, grain=CLOCK_SUBCOMPONENT )
 
-#ifdef MOVING_NEST
-   id_fv_tracker= mpp_clock_id ('FV tracker',  flags = clock_flag_default, grain=CLOCK_SUBCOMPONENT )
-#endif
                     call timing_off('ATMOS_INIT')
 
    ! Do CCPP fast physics initialization before call to adiabatic_init (since this calls fv_dynamics)
@@ -961,14 +944,11 @@ end subroutine get_nth_domain_info
 !! decomposition for the current cubed-sphere tile.
 !>@detail Coupling is done using the mass/temperature grid with no halos.
  subroutine atmosphere_domain ( fv_domain, rd_domain, layout, regional, nested, &
-                                moving_nest_parent, is_moving_nest, &
                                 ngrids_atmos, mygrid_atmos, pelist )
    type(domain2d), intent(out) :: fv_domain, rd_domain
    integer, intent(out) :: layout(2)
    logical, intent(out) :: regional
    logical, intent(out) :: nested
-   logical, intent(out) :: moving_nest_parent
-   logical, intent(out) :: is_moving_nest
    integer, intent(out) :: ngrids_atmos
    integer, intent(out) :: mygrid_atmos
    integer, pointer, intent(out) :: pelist(:)
@@ -985,31 +965,6 @@ subroutine atmosphere_domain ( fv_domain, rd_domain, layout, regional, nested, &
    call set_atmosphere_pelist()
    pelist => Atm(mygrid)%pelist
 
-   moving_nest_parent = .false.
-   is_moving_nest = .false.
-
-#ifdef MOVING_NEST
-   ! Currently, the moving nesting configuration only supports one parent (global
-   ! or regional) with one moving nest.
-   ! This will need to be revisited when multiple and telescoping moving nests are enabled.
-
-   ! Set is_moving_nest to true if this is a moving nest
-   is_moving_nest = Moving_nest(mygrid)%mn_flag%is_moving_nest
-   ! Set parent_of_moving_nest to true if it has a moving nest child
-   !do n=1,ngrids
-   !  print '("[INFO] WDR atmosphere_domain npe=",I0," mygrid=",I0," n=",I0," is_moving_nest=",L1)', mpp_pe(), mygrid, n, Moving_nest(n)%mn_flag%is_moving_nest
-   !enddo
-
-   do n=2,ngrids
-     if ( mygrid == Atm(n)%parent_grid%grid_number .and. &
-          Moving_nest(n)%mn_flag%is_moving_nest ) then
-       moving_nest_parent = .true.
-     endif
-   enddo
-   !print '("[INFO] WDR atmosphere_domain npe=",I0," moving_nest_parent=",L1," is_moving_nest=",L1)', mpp_pe(), moving_nest_parent, is_moving_nest
-
-#endif
-
  end subroutine atmosphere_domain
 
 
@@ -1791,29 +1746,6 @@ subroutine atmosphere_state_update (Time, IPD_Data, IAU_Data, Atm_block, flip_vc
      call mpp_clock_end(id_fv_diag)
    endif
 
-#ifdef MOVING_NEST
-  !---- FV internal vortex tracker -----
-   if ( Moving_nest(mygrid)%mn_flag%is_moving_nest ) then
-   if ( Moving_nest(mygrid)%mn_flag%vortex_tracker .eq. 2 .or. &
-        Moving_nest(mygrid)%mn_flag%vortex_tracker .eq. 6 .or. &
-        Moving_nest(mygrid)%mn_flag%vortex_tracker .eq. 7 ) then
-
-   fv_time = Time_next
-   call get_time (fv_time, seconds,  days)
-   call get_time (Time_step_atmos, sec)
-   if (mod(seconds,Moving_nest(mygrid)%mn_flag%ntrack*sec) .eq. 0) then
-     call mpp_clock_begin(id_fv_tracker)
-     call timing_on('FV_TRACKER')
-     call fv_diag_tracker(Atm(mygrid:mygrid), zvir, fv_time)
-     call fv_tracker_center(Atm(mygrid), mygrid, fv_time)
-     call timing_off('FV_TRACKER')
-     call mpp_clock_end(id_fv_tracker)
-   endif
-
-   endif
-   endif
-#endif
-
  end subroutine atmosphere_state_update
 
 

diff --git a/model/fv_arrays.F90 b/model/fv_arrays.F90
@@ -748,6 +748,8 @@ module fv_arrays_mod
                                           !< is recommended to only set this to .true. when initializing the model.
    logical :: fv_debug  = .false.  !< Whether to turn on additional diagnostics in fv_dynamics.
                                    !< The default is .false.
+   logical :: fv_timers  = .false. !< Whether to turn on performance metering timers in the dycore and moving nest
+                                   !< The default is .false.
    logical :: srf_init  = .false.
    logical :: mountain  = .true.  !< Takes topography into account when initializing the
                                   !< model. Set this to .true. to apply the terrain filter (if n_zs_filter = 2

diff --git a/model/fv_control.F90 b/model/fv_control.F90
@@ -161,11 +161,6 @@ module fv_control_mod
    use molecular_diffusion_mod,     only: molecular_diffusion_init, &
                                           read_namelist_molecular_diffusion_nml
 
-#ifdef MOVING_NEST
-   use fv_moving_nest_types_mod, only: fv_moving_nest_init, deallocate_fv_moving_nests
-   use fv_tracker_mod,           only: deallocate_tracker
-#endif
-
    implicit none
    private
 
@@ -355,6 +350,7 @@ subroutine fv_control_init(Atm, dt_atmos, this_grid, grids_on_this_pe, p_split,
      logical , pointer :: reproduce_sum
      logical , pointer :: adjust_dry_mass
      logical , pointer :: fv_debug
+     logical , pointer :: fv_timers
      logical , pointer :: srf_init
      logical , pointer :: mountain
      logical , pointer :: remap_t
@@ -408,7 +404,7 @@ subroutine fv_control_init(Atm, dt_atmos, this_grid, grids_on_this_pe, p_split,
      integer, pointer :: nrows_blend
      logical, pointer :: regional_bcs_from_gsi
      logical, pointer :: write_restart_with_bcs
-     integer, pointer :: parent_tile, refinement, nestbctype, nestupdate, nsponge, ioffset, joffset
+     integer, pointer :: parent_tile, refinement, nestbctype, nestupdate, upoff, nsponge, ioffset, joffset
      real, pointer :: s_weight, update_blend
 
      character(len=16), pointer :: restart_resolution
@@ -542,13 +538,6 @@ subroutine fv_control_init(Atm, dt_atmos, this_grid, grids_on_this_pe, p_split,
         endif
      enddo
 
-#ifdef MOVING_NEST
-     ! This has to be called on the input.nml namelist for all PEs
-     !   input_nest02.nml does not have any of the moving nest parameters
-     !   Later call to read_input_nml changes which namelist is used
-     call fv_moving_nest_init(Atm)
-#endif
-
      if (pecounter /= npes) then
         if (mpp_pe() == 0) then
            print*, 'npes = ', npes, ', grid_pes = ', grid_pes(1:ngrids)
@@ -917,6 +906,7 @@ subroutine set_namelist_pointers(Atm)
        reproduce_sum                 => Atm%flagstruct%reproduce_sum
        adjust_dry_mass               => Atm%flagstruct%adjust_dry_mass
        fv_debug                      => Atm%flagstruct%fv_debug
+       fv_timers                     => Atm%flagstruct%fv_timers
        srf_init                      => Atm%flagstruct%srf_init
        mountain                      => Atm%flagstruct%mountain
        remap_t                       => Atm%flagstruct%remap_t
@@ -971,6 +961,7 @@ subroutine set_namelist_pointers(Atm)
        refinement                    => Atm%neststruct%refinement
        nestbctype                    => Atm%neststruct%nestbctype
        nestupdate                    => Atm%neststruct%nestupdate
+       upoff                         => Atm%neststruct%upoff
        nsponge                       => Atm%neststruct%nsponge
        s_weight                      => Atm%neststruct%s_weight
        ioffset                       => Atm%neststruct%ioffset
@@ -1056,7 +1047,7 @@ subroutine read_namelist_fv_core_nml(Atm)
             use_logp, p_fac, a_imp, k_split, n_split, m_split, q_split, print_freq, write_3d_diags, &
             do_schmidt, do_cube_transform, &
             hord_mt, hord_vt, hord_tm, hord_dp, hord_tr, shift_fac, stretch_fac, target_lat, target_lon, &
-            kord_mt, kord_wz, kord_tm, kord_tr, fv_debug, fv_land, nudge, do_sat_adj, do_inline_mp, do_f3d, &
+            kord_mt, kord_wz, kord_tm, kord_tr, fv_debug, fv_timers, fv_land, nudge, do_sat_adj, do_inline_mp, do_f3d, &
             external_ic, read_increment, ncep_ic, nggps_ic, hrrrv3_ic, ecmwf_ic, use_new_ncep, use_ncep_phy, fv_diag_ic, &
             external_eta, res_latlon_dynamics, res_latlon_tracers, scale_z, w_max, z_min, lim_fac, &
             dddmp, d2_bg, d4_bg, vtdm4, trdm2, d_ext, delt_max, beta, non_ortho, n_sponge, &
@@ -1071,7 +1062,7 @@ subroutine read_namelist_fv_core_nml(Atm)
             deglon_start, deglon_stop, deglat_start, deglat_stop, &
             phys_hydrostatic, use_hydro_pressure, make_hybrid_z, old_divg_damp, add_noise, butterfly_effect, &
             molecular_diffusion, dz_min, psm_bc, nested, twowaynest, nudge_qv, &
-            nestbctype, nestupdate, nsponge, s_weight, &
+            nestbctype, nestupdate, upoff, nsponge, s_weight, &
             check_negative, nudge_ic, halo_update_type, gfs_phil, agrid_vel_rst,     &
             do_uni_zfull, adj_mass_vmr, fac_n_spl, fhouri, update_blend, regional, bc_update_interval,  &
             regional_bcs_from_gsi, write_restart_with_bcs, nrows_blend,  &
@@ -1336,11 +1327,6 @@ subroutine fv_end(Atm, this_grid, restart_endfcst)
        call deallocate_coarse_restart_type(Atm(n)%coarse_graining%restart)
     end do
 
-#ifdef MOVING_NEST
-    call deallocate_fv_moving_nests(ngrids)
-    call deallocate_tracker(ngrids)
-#endif
-
  end subroutine fv_end
 !-------------------------------------------------------------------------------
 

diff --git a/model/fv_grid_utils.F90 b/model/fv_grid_utils.F90
@@ -902,8 +902,9 @@ subroutine grid_utils_init(Atm, npx, npy, npz, non_ortho, grid_type, c2l_order)
 
 #ifndef MOVING_NEST
 ! WDR Need to use these arrays again if moving the nest
-!     So don't deallocate them.
+!     So don't want to deallocate them here.
 !     TODO clean them up at end of model run for completeness
+! These are deallocated in fv_moving_nest_main.F90::moving_nest_end()
 
 !--- deallocate the higher-order gridstruct arrays
 !rab      deallocate ( Atm%gridstruct%grid_64 )

diff --git a/model/fv_nesting.F90 b/model/fv_nesting.F90
@@ -159,13 +159,6 @@ subroutine dealloc_nested_buffers(Atm)
 
   integer :: n, ncnst
   !logical :: dummy = .false.
-  logical :: debug_log = .false.
-
-  integer :: this_pe
-
-  this_pe = mpp_pe()
-
-  if (debug_log) print '("[INFO] WDR NBC deallocating buffers fv_nesting.F90 npe=",I0)', this_pe
 
   call deallocate_fv_nest_BC_type(u_buf)
   call deallocate_fv_nest_BC_type(v_buf)