Merge branch 'NOAA-EMC:develop' into develop

.github/workflows/GCC.yml

@@ -15,33 +15,34 @@ on:
     - develop
 
 jobs:
-  GCC:
+  build_spack:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        cmake_opts: ["-D32BIT=ON", "-D32BIT=OFF"]
-        gcc_ver: ["11"]
-        mpi: ["mpich"]
+        gcc_ver: ["12"]
+        mpi: ["mpich", "openmpi"]
 
     steps:
 
-    - name: install-doxygen
-      run: |
-        sudo apt-get install doxygen graphviz
-        
     - name: checkout-fv3atm
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: ${{ github.workspace }}/fv3atm
         submodules: recursive
 
+    - name: install-cmake
+      run: |
+        cd ${{ github.workspace }}
+        curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx
+        echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH
+
     - name: cache-spack
       id: cache-spack
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ github.workspace }}/spack-develop
-        key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-2-${{ matrix.cmake_opts }}-${{ matrix.mpich }}
+        key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}
 
     # Building dependencies takes 40+ min
     - name: spack-install
@@ -58,6 +59,42 @@ jobs:
         spack config add "packages:mpi:require:'${{ matrix.mpi }}'"
         spack concretize |& tee ${SPACK_ENV}/log.concretize
         spack install -j2 --fail-fast
+        spack clean --all
+
+  build_fv3atm:
+    needs: build_spack
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        cmake_opts: ["-D32BIT=ON", "-D32BIT=OFF"]
+        gcc_ver: ["12"]
+        mpi: ["mpich", "openmpi"]
+
+    steps:
+
+    - name: install-doxygen
+      run: |
+        sudo apt-get install doxygen graphviz
+
+    - name: install-cmake
+      run: |
+        cd ${{ github.workspace }}
+        curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx
+        echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH
+
+    - name: checkout-fv3atm
+      uses: actions/checkout@v4
+      with:
+        path: ${{ github.workspace }}/fv3atm
+        submodules: recursive
+
+    - name: cache-spack
+      id: cache-spack
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ github.workspace }}/spack-develop
+        key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}
 
     - name: build-fv3atm
       run: |
@@ -68,22 +105,25 @@ jobs:
         git clone https://github.com/NOAA-EMC/CMakeModules
         git clone --recurse-submodules https://github.com/NOAA-PSL/stochastic_physics stochastic_physics_repo
         mkdir ${GITHUB_WORKSPACE}/build
-        sed -i 's/doc /upp_doc /' upp/docs/CMakeLists.txt
         cd ${GITHUB_WORKSPACE}/build
+        export CC=mpicc
+        export CXX=mpicxx
+        export FC=mpif90
+        cat /home/runner/work/fv3atm/fv3atm/spack-develop/opt/spack/linux-ubuntu22.04-zen2/gcc-12.3.0/fms-2023.04-*/lib/cmake/fms/fms-config.cmake
         cmake ${GITHUB_WORKSPACE}/fv3atm -DBUILD_TESTING=ON ${{ matrix.cmake_opts }} -DENABLE_DOCS=ON
         make -j2
         ls -l /home/runner/work/fv3atm/fv3atm/fv3atm/io
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
-        name: docs
+        name: docs-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}-${{ matrix.cmake_opts }}
         path: |
           build/docs/html
-          
+
     - name: debug-artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: ${{ failure() }}
       with:
-        name: ccpp_prebuild_logs
+        name: ccpp_prebuild_logs-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}-${{ matrix.cmake_opts }}
         path: ${{ github.workspace }}/build/ccpp/ccpp_prebuild.*
 

CMakeLists.txt

@@ -9,7 +9,7 @@ option(ENABLE_DOCS "Enable generation of doxygen-based documentation." OFF)
 # Determine whether or not to generate documentation.
 if(ENABLE_DOCS)
   find_package(Doxygen REQUIRED)
-  add_subdirectory(docs)  
+  add_subdirectory(docs)
 endif()
 
 # Enable CI build & unit testing:
@@ -37,7 +37,11 @@ set(USE_GFSL63  ON)
 if(MOVING_NEST)
   set(MOVING_NEST ON)
 endif()
+if(HYDRO)
+  set(HYDRO ON)
+endif()
 add_subdirectory(atmos_cubed_sphere)
+target_compile_definitions(fv3 PRIVATE BYPASS_BREED_SLP_INLINE)
 
 ###############################################################################
 ### fv3atm
@@ -60,7 +64,11 @@ if(NOT PARALLEL_NETCDF)
 endif()
 
 if(MOVING_NEST)
-  list(APPEND _fv3atm_defs_private MOVING_NEST MOIST_CAPPA USE_COND)
+  list(APPEND _fv3atm_defs_private MOVING_NEST)
+  if(NOT HYDRO)
+    list(APPEND _fv3atm_defs_private MOIST_CAPPA USE_COND)
+  endif()
+
   if(DEBUG)
     list(APPEND _fv3atm_defs_private DEBUG)
   endif()

atmos_model.F90

@@ -533,12 +533,12 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
   type (time_type), intent(in) :: Time_init, Time, Time_step
 !--- local variables ---
   integer :: unit, i
-  integer :: mlon, mlat, nlon, nlat, nlev, sec
+  integer :: mlon, mlat, nlon, nlat, nlev, sec, sec_lastfhzerofh
   integer :: ierr, io, logunit
   integer :: tile_num
   integer :: isc, iec, jsc, jec
   real(kind=GFS_kind_phys) :: dt_phys
-  logical              :: p_hydro, hydro
+  logical              :: p_hydro, hydro, tmpflag_fhzero
   logical, save        :: block_message = .true.
   type(GFS_init_type)  :: Init_parm
   integer              :: bdat(8), cdat(8)
@@ -582,7 +582,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
    call atmosphere_diag_axes (Atmos%axes)
    call atmosphere_etalvls (Atmos%ak, Atmos%bk, flip=flip_vc)
 
-   call atmosphere_control_data (isc, iec, jsc, jec, nlev, p_hydro, hydro, tile_num)
+   tile_num=-1
+   call atmosphere_control_data (isc, iec, jsc, jec, nlev, p_hydro, hydro, global_tile_num=tile_num)
 
    allocate (Atmos%lon(nlon,nlat), Atmos%lat(nlon,nlat))
    call atmosphere_grid_ctr (Atmos%lon, Atmos%lat)
@@ -788,8 +789,33 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
    !--- WARNING: For special cases that model needs to restart at non-multiple of fhzero
    !--- the fields in first output files are not accumulated from the beginning of
    !--- the bucket, but the restart time.
-   if (mod(sec,int(GFS_Control%fhzero*3600.)) /= 0) then
-     diag_time = Time - real_to_time_type(mod(int((GFS_Control%kdt - 1)*dt_phys/3600.),int(GFS_Control%fhzero))*3600.0)
+   if( GFS_Control%fhzero_array(1) > 0. ) then
+     fhzero_loop: do i=1,size(GFS_Control%fhzero_array)
+       tmpflag_fhzero= .false.
+       if( GFS_Control%fhzero_array(i) > 0.) then
+         if( i == 1 ) then
+           if( sec <= GFS_Control%fhzero_fhour(i)*3600. ) tmpflag_fhzero = .true.
+         else if( i > 1 ) then
+           if( sec > GFS_Control%fhzero_fhour(i-1)*3600. .and. sec <=GFS_Control%fhzero_fhour(i)*3600. ) &
+             tmpflag_fhzero = .true.
+         endif
+         if( tmpflag_fhzero ) then
+           GFS_Control%fhzero = GFS_Control%fhzero_array(i)
+           if( GFS_Control%fhzero > 0) then
+             sec_lastfhzerofh = (int(sec/3600.)/int(GFS_Control%fhzero))*int(GFS_Control%fhzero)*3600
+           else
+             sec_lastfhzerofh = 0
+           endif
+         endif
+       endif
+     enddo fhzero_loop
+   else
+     sec_lastfhzerofh = 0
+   endif
+   if (mpp_pe() == mpp_root_pe()) print *,'in atmos_model, fhzero=',GFS_Control%fhzero, 'fhour=',sec/3600.,sec_lastfhzerofh/3600
+
+   if (mod((sec-sec_lastfhzerofh),int(GFS_Control%fhzero*3600.)) /= 0) then
+     diag_time = Time - real_to_time_type(mod(int((GFS_Control%kdt - 1)*dt_phys-sec_lastfhzerofh),int(GFS_Control%fhzero))*3600.0)
      if (mpp_pe() == mpp_root_pe()) print *,'Warning: in atmos_init,start at non multiple of fhzero'
    endif
    if (Atmos%iau_offset > zero) then
@@ -948,8 +974,9 @@ subroutine update_atmos_model_state (Atmos, rc)
   type (atmos_data_type), intent(inout) :: Atmos
   integer, optional,      intent(out)   :: rc
 !--- local variables
-  integer :: localrc
+  integer :: i, localrc, sec_lastfhzerofh
   integer :: isec, seconds, isec_fhzero
+  logical :: tmpflag_fhzero
   real(kind=GFS_kind_phys) :: time_int, time_intfull
 !
     if (present(rc)) rc = ESMF_SUCCESS
@@ -1000,16 +1027,38 @@ subroutine update_atmos_model_state (Atmos, rc)
                             GFS_control%levs, 1, 1, 1.0_GFS_kind_phys, time_int, time_intfull, &
                             GFS_control%fhswr, GFS_control%fhlwr)
     endif
-    if (nint(GFS_control%fhzero) > 0) then
-      if (mod(isec,3600*nint(GFS_control%fhzero)) == 0) diag_time = Atmos%Time
+
+    !---  find current fhzero
+    if( GFS_Control%fhzero_array(1) > 0. ) then
+      fhzero_loop: do i=1,size(GFS_Control%fhzero_array)
+        tmpflag_fhzero = .false.
+        if( GFS_Control%fhzero_array(i) > 0.) then
+          if( i == 1 ) then
+            if( seconds <= GFS_Control%fhzero_fhour(i)*3600. ) tmpflag_fhzero = .true.
+          else if( i > 1 ) then
+            if( seconds > GFS_Control%fhzero_fhour(i-1)*3600. .and. seconds <= GFS_Control%fhzero_fhour(i)*3600. ) &
+              tmpflag_fhzero = .true.
+          endif
+          if( tmpflag_fhzero) then
+            GFS_Control%fhzero = GFS_Control%fhzero_array(i)
+            if( GFS_Control%fhzero > 0) then
+              sec_lastfhzerofh = (int(seconds/3600.)/int(GFS_Control%fhzero))*int(GFS_Control%fhzero)*3600
+            else
+              sec_lastfhzerofh = 0
+            endif
+          endif
+        endif
+      enddo fhzero_loop
     else
-      if (mod(isec,nint(3600*GFS_control%fhzero)) == 0) diag_time = Atmos%Time
+      sec_lastfhzerofh = 0
     endif
-    call diag_send_complete_instant (Atmos%Time)
+    if (mpp_pe() == mpp_root_pe()) print *,'in atmos_model update, fhzero=',GFS_Control%fhzero, 'fhour=',seconds/3600.,sec_lastfhzerofh/3600.
 
-
-    !--- this may not be necessary once write_component is fully implemented
-    !!!call diag_send_complete_extra (Atmos%Time)
+    if (nint(GFS_Control%fhzero) > 0) then
+      if (mod(isec - sec_lastfhzerofh,nint(GFS_Control%fhzero*3600.)) == 0) diag_time = Atmos%Time
+!    if (mpp_pe() == mpp_root_pe()) print *,'in atmos_model update time=',isec/3600.,'last fhzeo=',sec_lastfhzerofh
+    endif
+    call diag_send_complete_instant (Atmos%Time)
 
     !--- get bottom layer data from dynamical core for coupling
     call atmosphere_get_bottom_layer (Atm_block, DYCORE_Data)

ccpp/CMakeLists.txt

@@ -84,8 +84,10 @@ endif(CCPP_32BIT)
 
 #------------------------------------------------------------------------------
 # Add model-specific flags for C/C++/Fortran preprocessor
-add_definitions(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM)
-add_definitions(-DINTERNAL_FILE_NML)
+if(NOT HYDRO)
+  add_definitions(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM)
+endif()
+add_definitions(-DINTERNAL_FILE_NML -DNEMS_GSM)
 
 if(MULTI_GASES)
   add_definitions(-DMULTI_GASES)

ccpp/data/CCPP_typedefs.F90

@@ -452,10 +452,18 @@ module CCPP_typedefs
      integer                             :: ie
      integer                             :: isd
      integer                             :: ied
+     integer                             :: isc1
+     integer                             :: iec1
+     integer                             :: isc2
+     integer                             :: iec2
      integer                             :: js
      integer                             :: je
      integer                             :: jsd
      integer                             :: jed
+     integer                             :: jsc1
+     integer                             :: jec1
+     integer                             :: jsc2
+     integer                             :: jec2
      integer                             :: ng
      integer                             :: npz
      integer                             :: npzp1
@@ -1557,6 +1565,18 @@ subroutine gfdl_interstitial_create (Interstitial, is, ie, isd, ied, js, je, jsd
     integer,        intent(in)           :: mpirank
     integer,        intent(in)           :: mpiroot
     !
+    integer :: isc1, jsc1, iec1, jec1
+    integer :: isc2, jsc2, iec2, jec2
+    !
+    isc1 = lbound(delp, dim=1)
+    jsc1 = lbound(delp, dim=2)
+    iec1 = ubound(delp, dim=1)
+    jec1 = ubound(delp, dim=2)
+    isc2 = lbound(delz, dim=1)
+    jsc2 = lbound(delz, dim=2)
+    iec2 = ubound(delz, dim=1)
+    jec2 = ubound(delz, dim=2)
+    !
 #ifdef MOIST_CAPPA
     Interstitial%npzcappa = npz
     allocate (Interstitial%cappa  (isd:ied, jsd:jed, 1:npz) )
@@ -1594,13 +1614,22 @@ subroutine gfdl_interstitial_create (Interstitial, is, ie, isd, ied, js, je, jsd
     Interstitial%ie         =  ie
     Interstitial%isd        =  isd
     Interstitial%ied        =  ied
+    Interstitial%isc1       =  isc1
+    Interstitial%iec1       =  iec1
+    Interstitial%isc2       =  isc2
+    Interstitial%iec2       =  iec2
     Interstitial%js         =  js
     Interstitial%je         =  je
     Interstitial%jsd        =  jsd
     Interstitial%jed        =  jed
+    Interstitial%jsc1       =  jsc1
+    Interstitial%jec1       =  jec1
+    Interstitial%jsc2       =  jsc2
+    Interstitial%jec2       =  jec2
     Interstitial%ng         =  ng
     Interstitial%npz        =  npz
     Interstitial%npzp1      =  npz+1
+    !
     ! Set up links from GFDL_interstitial DDT to ATM DDT
     Interstitial%delp       => delp
     Interstitial%delz       => delz
@@ -1617,6 +1646,7 @@ subroutine gfdl_interstitial_create (Interstitial, is, ie, isd, ied, js, je, jsd
     if (do_qs) Interstitial%qs => qs
     if (do_qg) Interstitial%qg => qg
     if (do_qa) Interstitial%qc => qc
+    !
 #ifdef USE_COND
     Interstitial%npzq_con = npz
 #else