From 383687ef8752af2408d882c47a4f3cbb4d3121ee Mon Sep 17 00:00:00 2001 From: JONG KIM Date: Thu, 4 Apr 2024 16:34:52 -0400 Subject: [PATCH 01/10] cubed_sphere hash fix to f060e85 (#817) * Update cubed_sphere@f060e85 hash: a bug fix in the HAILCAST diagnostic code (units issue) --- atmos_cubed_sphere | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 6663459e5..f060e857f 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 6663459e58a04e3bda2157d5891d227e3abc3c7a +Subproject commit f060e857f184a4e8e988d8563794066525357336 From 37e7d4859db4eb75472091abc650831060037715 Mon Sep 17 00:00:00 2001 From: "Samuel Trahan (NOAA contractor)" <39415369+SamuelTrahanNOAA@users.noreply.github.com> Date: Sun, 14 Apr 2024 13:00:07 -0400 Subject: [PATCH 02/10] bug fixes: kchunk3d ignored, hailwat uninitialized in dycore, tile_num wrong for nests (#806) * nesting fixes 1. Initialize hailwat index in dynamical core. 2. Use the correct tile number for nests in atmos_model.F90. * remove unneeded write statements * Fix kchunk3d in io/module_write_netcdf.F90 --------- Co-authored-by: Dusan Jovic --- atmos_cubed_sphere | 2 +- atmos_model.F90 | 3 ++- io/module_write_netcdf.F90 | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index f060e857f..0301022fc 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit f060e857f184a4e8e988d8563794066525357336 +Subproject commit 0301022fc73b23f20b42b52e999fa47752708ef0 diff --git a/atmos_model.F90 b/atmos_model.F90 index 91a566671..a2ca32a75 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -582,7 +582,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) call atmosphere_diag_axes (Atmos%axes) call atmosphere_etalvls (Atmos%ak, Atmos%bk, flip=flip_vc) - call atmosphere_control_data (isc, iec, jsc, jec, nlev, p_hydro, hydro, tile_num) + tile_num=-1 + call atmosphere_control_data (isc, iec, jsc, jec, nlev, p_hydro, hydro, global_tile_num=tile_num) allocate (Atmos%lon(nlon,nlat), Atmos%lat(nlon,nlat)) call atmosphere_grid_ctr (Atmos%lon, Atmos%lat) diff --git a/io/module_write_netcdf.F90 b/io/module_write_netcdf.F90 index 2b5fcacc1..b0164151e 100644 --- a/io/module_write_netcdf.F90 +++ b/io/module_write_netcdf.F90 @@ -407,7 +407,7 @@ end function nf_set_log_level if (is_cubed_sphere) then chunksizes = [im, jm, lm, tileCount, 1] else - chunksizes = [ichunk3d(grid_id), jchunk3d(grid_id), fldlev(i), 1] + chunksizes = [ichunk3d(grid_id), jchunk3d(grid_id), min(kchunk3d(grid_id),fldlev(i)), 1] end if ncerr = nf90_def_var_chunking(ncid, varids(i), NF90_CHUNKED, chunksizes) ; NC_ERR_STOP(ncerr) end if From 979bcab28f63b37411698cd9d23d04d0b0fe3a7e Mon Sep 17 00:00:00 2001 From: Dusan Jovic <48258889+DusanJovic-NOAA@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:42:06 -0400 Subject: [PATCH 03/10] Update dycore to remove compiler warnings (#813) * Update atmos_cubed_sphere and add compile definition BYPASS_BREED_SLP_INLINE to dycore * Pass correct mpi communicator type to post_alctvars routine in inline post --- CMakeLists.txt | 3 ++- atmos_cubed_sphere | 2 +- io/post_fv3.F90 | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d76a8f25..c310110ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ option(ENABLE_DOCS "Enable generation of doxygen-based documentation." OFF) # Determine whether or not to generate documentation. if(ENABLE_DOCS) find_package(Doxygen REQUIRED) - add_subdirectory(docs) + add_subdirectory(docs) endif() # Enable CI build & unit testing: @@ -38,6 +38,7 @@ if(MOVING_NEST) set(MOVING_NEST ON) endif() add_subdirectory(atmos_cubed_sphere) +target_compile_definitions(fv3 PRIVATE BYPASS_BREED_SLP_INLINE) ############################################################################### ### fv3atm diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 0301022fc..97114888f 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 0301022fc73b23f20b42b52e999fa47752708ef0 +Subproject commit 97114888f529c2df70ed4a350fd2428df74839ff diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index e00accfa0..c0ba56dda 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -136,7 +136,7 @@ subroutine post_run_fv3(wrt_int_state,grid_id,mype,mpicomp,lead_write, & wrt_int_state%out_grid_info(grid_id)%jm, & wrt_int_state%out_grid_info(grid_id)%lm, & mype,wrttasks_per_group,lead_write, & - mpicomp,jts,jte,jstagrp,jendgrp,its,ite,istagrp,iendgrp) + mpicomp%mpi_val,jts,jte,jstagrp,jendgrp,its,ite,istagrp,iendgrp) ! !----------------------------------------------------------------------- !*** read namelist for pv,th,po From da95cc428d8b626e99250fd57a4279b4980044f8 Mon Sep 17 00:00:00 2001 From: WenMeng-NOAA <48260754+WenMeng-NOAA@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:36:43 -0400 Subject: [PATCH 04/10] Update upp submodule (#811) * Update github workflow. Update gcc to 12 and mpi to openmpi * Add 'spack clean' to reduce the size of the cache files * Split GCC.yml workflow into two jobs, build_spack and build_fv3atm * Update github workflow. Add mpich * Update github actions to v4 * Use action/cache/restore in build_fv3atm job * update upp revision to 5faac75 * Update GCC.yml --------- Co-authored-by: Dusan Jovic --- .github/workflows/GCC.yml | 73 ++++++++++++++++++++++++++++++--------- upp | 2 +- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/.github/workflows/GCC.yml b/.github/workflows/GCC.yml index 4f1b7a894..0da81e763 100644 --- a/.github/workflows/GCC.yml +++ b/.github/workflows/GCC.yml @@ -15,33 +15,34 @@ on: - develop jobs: - GCC: + build_spack: runs-on: ubuntu-latest strategy: matrix: - cmake_opts: ["-D32BIT=ON", "-D32BIT=OFF"] - gcc_ver: ["11"] - mpi: ["mpich"] + gcc_ver: ["12"] + mpi: ["mpich", "openmpi"] steps: - - name: install-doxygen - run: | - sudo apt-get install doxygen graphviz - - name: checkout-fv3atm - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: ${{ github.workspace }}/fv3atm submodules: recursive + - name: install-cmake + run: | + cd ${{ github.workspace }} + curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx + echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH + - name: cache-spack id: cache-spack - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ github.workspace }}/spack-develop - key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-2-${{ matrix.cmake_opts }}-${{ matrix.mpich }} + key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }} # Building dependencies takes 40+ min - name: spack-install @@ -58,6 +59,42 @@ jobs: spack config add "packages:mpi:require:'${{ matrix.mpi }}'" spack concretize |& tee ${SPACK_ENV}/log.concretize spack install -j2 --fail-fast + spack clean --all + + build_fv3atm: + needs: build_spack + runs-on: ubuntu-latest + + strategy: + matrix: + cmake_opts: ["-D32BIT=ON", "-D32BIT=OFF"] + gcc_ver: ["12"] + mpi: ["mpich", "openmpi"] + + steps: + + - name: install-doxygen + run: | + sudo apt-get install doxygen graphviz + + - name: install-cmake + run: | + cd ${{ github.workspace }} + curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx + echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH + + - name: checkout-fv3atm + uses: actions/checkout@v4 + with: + path: ${{ github.workspace }}/fv3atm + submodules: recursive + + - name: cache-spack + id: cache-spack + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }}/spack-develop + key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }} - name: build-fv3atm run: | @@ -70,20 +107,24 @@ jobs: mkdir ${GITHUB_WORKSPACE}/build sed -i 's/doc /upp_doc /' upp/docs/CMakeLists.txt cd ${GITHUB_WORKSPACE}/build + export CC=mpicc + export CXX=mpicxx + export FC=mpif90 + cat /home/runner/work/fv3atm/fv3atm/spack-develop/opt/spack/linux-ubuntu22.04-zen2/gcc-12.3.0/fms-2023.04-*/lib/cmake/fms/fms-config.cmake cmake ${GITHUB_WORKSPACE}/fv3atm -DBUILD_TESTING=ON ${{ matrix.cmake_opts }} -DENABLE_DOCS=ON make -j2 ls -l /home/runner/work/fv3atm/fv3atm/fv3atm/io - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: docs + name: docs-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}-${{ matrix.cmake_opts }} path: | build/docs/html - + - name: debug-artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ failure() }} with: - name: ccpp_prebuild_logs + name: ccpp_prebuild_logs-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}-${{ matrix.cmake_opts }} path: ${{ github.workspace }}/build/ccpp/ccpp_prebuild.* diff --git a/upp b/upp index 945cb2cef..5faac752d 160000 --- a/upp +++ b/upp @@ -1 +1 @@ -Subproject commit 945cb2cef5e8bd5949afd4f0fc35c4fb6e95a1bf +Subproject commit 5faac752d9550d3570705358fa1eb3f5ac78a786 From 683061b3fd58716625eed61d7a7143e985f733d4 Mon Sep 17 00:00:00 2001 From: AnningCheng-NOAA <48297505+AnningCheng-NOAA@users.noreply.github.com> Date: Wed, 1 May 2024 17:06:05 -0400 Subject: [PATCH 05/10] Fixed excessive evaporation when both innerloop=T and mraerosol=T (#816) * update ccpp/framework * fixed excessive evaporation when both innerloop=T and mraerosol=T --------- Co-authored-by: Grant Firl --- ccpp/framework | 2 +- ccpp/physics | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ccpp/framework b/ccpp/framework index 011db4f80..741212e49 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit 011db4f80a02cba6d65958ace56e8efb197be62b +Subproject commit 741212e4962d371520f773e2be9481142f79635e diff --git a/ccpp/physics b/ccpp/physics index 9b0ac7b16..1c2a898e9 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 9b0ac7b16a45afe5e7f1abf9571d3484158a5b43 +Subproject commit 1c2a898e97f069eeeb930cfb542db67ed349b72c From 7147dac3d5075c38a43cc221d1e7e7337c28437f Mon Sep 17 00:00:00 2001 From: Dusan Jovic <48258889+DusanJovic-NOAA@users.noreply.github.com> Date: Thu, 9 May 2024 09:35:51 -0400 Subject: [PATCH 06/10] Add option to specify history and inline post files output directory (#826) * Add model_configure option to specify history file output directory (fv3atm_output_dir) * Update workflows/GCC.yml * Move reading of 'fv3atm_output_dir' config option to module_wrt_grid_comp * Update upp --- .github/workflows/GCC.yml | 5 +-- io/module_fv3_io_def.F90 | 60 ++++++++++++++-------------- io/module_wrt_grid_comp.F90 | 21 +++++++--- io/post_fv3.F90 | 80 +++++++++++++++++++------------------ upp | 2 +- 5 files changed, 90 insertions(+), 78 deletions(-) diff --git a/.github/workflows/GCC.yml b/.github/workflows/GCC.yml index 0da81e763..5f3891684 100644 --- a/.github/workflows/GCC.yml +++ b/.github/workflows/GCC.yml @@ -32,7 +32,7 @@ jobs: submodules: recursive - name: install-cmake - run: | + run: | cd ${{ github.workspace }} curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH @@ -78,7 +78,7 @@ jobs: sudo apt-get install doxygen graphviz - name: install-cmake - run: | + run: | cd ${{ github.workspace }} curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH @@ -105,7 +105,6 @@ jobs: git clone https://github.com/NOAA-EMC/CMakeModules git clone --recurse-submodules https://github.com/NOAA-PSL/stochastic_physics stochastic_physics_repo mkdir ${GITHUB_WORKSPACE}/build - sed -i 's/doc /upp_doc /' upp/docs/CMakeLists.txt cd ${GITHUB_WORKSPACE}/build export CC=mpicc export CXX=mpicxx diff --git a/io/module_fv3_io_def.F90 b/io/module_fv3_io_def.F90 index ce600d8e1..05d7c5fbe 100644 --- a/io/module_fv3_io_def.F90 +++ b/io/module_fv3_io_def.F90 @@ -12,89 +12,91 @@ module module_fv3_io_def !> Number of processors used in the forecast grid component integer :: num_pes_fcst - !> Number of write tasks per write group. + !> Number of write tasks per write group. integer :: wrttasks_per_group !> Number of the write groups - integer :: write_groups + integer :: write_groups - !> Current write group - integer :: n_group + !> Current write group + integer :: n_group !> Number of history files - integer :: num_files + integer :: num_files !> Number of the ESMF field bundles for physics fields - integer :: nbdlphys + integer :: nbdlphys !> IAU running window length - integer :: iau_offset + integer :: iau_offset !> Logical variable to decide if full time (HH.MM.SS) is used in the history !! file names - logical :: lflname_fulltime + logical :: lflname_fulltime !> Logical variable to decide if unlimited time dimension is used - logical :: time_unlimited + logical :: time_unlimited + !> Directory name for model history output files + character(len=esmf_maxstr) :: fv3atm_output_dir !> Base names for model history output files - character(len=esmf_maxstr),dimension(:),allocatable :: filename_base + character(len=esmf_maxstr),dimension(:),allocatable :: filename_base - !> Output file format - character(len=esmf_maxstr),dimension(:),allocatable :: output_file + !> Output file format + character(len=esmf_maxstr),dimension(:),allocatable :: output_file !> The first write task in a write group - integer,dimension(:),allocatable :: lead_wrttask + integer,dimension(:),allocatable :: lead_wrttask !> The last write task in a write group - integer,dimension(:),allocatable :: last_wrttask + integer,dimension(:),allocatable :: last_wrttask !> Output grid type, e.g. "gaussian_grid" - character(len=esmf_maxstr),dimension(:),allocatable :: output_grid + character(len=esmf_maxstr),dimension(:),allocatable :: output_grid !> The i-dimension in the output grid - integer,dimension(:),allocatable :: imo + integer,dimension(:),allocatable :: imo !> The j-dimension in the output grid - integer,dimension(:),allocatable :: jmo + integer,dimension(:),allocatable :: jmo !> Longitude of the center point in the output grid - real,dimension(:),allocatable :: cen_lon + real,dimension(:),allocatable :: cen_lon !> Latitude of the center pointer in the output grid - real,dimension(:),allocatable :: cen_lat + real,dimension(:),allocatable :: cen_lat !> Longitude of the first grid point in the output grid - real,dimension(:),allocatable :: lon1 + real,dimension(:),allocatable :: lon1 !> Latitude of the first pointer in the output grid - real,dimension(:),allocatable :: lat1 + real,dimension(:),allocatable :: lat1 !> Longitude of the last grid point in the output grid - real,dimension(:),allocatable :: lon2 + real,dimension(:),allocatable :: lon2 !> Latitude of the last pointer in the output grid - real,dimension(:),allocatable :: lat2 + real,dimension(:),allocatable :: lat2 !> Longitude increment - real,dimension(:),allocatable :: dlon + real,dimension(:),allocatable :: dlon !> Latitude increment - real,dimension(:),allocatable :: dlat + real,dimension(:),allocatable :: dlat !> The first latitude from the pole at which the secant cone cuts the sphere - real,dimension(:),allocatable :: stdlat1 + real,dimension(:),allocatable :: stdlat1 !> The second latitude from the pole at which the secant cone cuts the sphere - real,dimension(:),allocatable :: stdlat2 + real,dimension(:),allocatable :: stdlat2 !> x-direction grid length - real,dimension(:),allocatable :: dx + real,dimension(:),allocatable :: dx !> y-direction grid length - real,dimension(:),allocatable :: dy + real,dimension(:),allocatable :: dy !> Deflate level to use, 0 means no deflate. integer,dimension(:),allocatable :: ideflate diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index eb3ccd8dc..5a3945714 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -28,13 +28,13 @@ module module_wrt_grid_comp ! use mpi_f08 use esmf - use fms_mod, only : uppercase - use fms - use mpp_mod, only : mpp_init, mpp_error + use fms, only : fms_init, fms_end, fms_mpp_uppercase, fms_mpp_error, FATAL + use fms, only : NO_CALENDAR, JULIAN, GREGORIAN, THIRTY_DAY_MONTHS, NOLEAP use write_internal_state use module_fv3_io_def, only : num_pes_fcst, & n_group, num_files, & + fv3atm_output_dir, & filename_base, output_grid, output_file, & imo,jmo,ichunk2d,jchunk2d, & ichunk3d,jchunk3d,kchunk3d, & @@ -282,6 +282,15 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return + call ESMF_ConfigGetAttribute(config=CF,value=fv3atm_output_dir, & + label ='fv3atm_output_dir:', default='./', rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + ! Make sure fv3atm_output_dir ends with '/' + if (fv3atm_output_dir(len(trim(fv3atm_output_dir)):len(trim(fv3atm_output_dir))) /= '/') then + fv3atm_output_dir = trim(fv3atm_output_dir) // '/' + end if + if( wrt_int_state%write_dopost ) then #ifdef INLINE_POST call ESMF_ConfigGetAttribute(config=CF,value=wrt_int_state%post_namelist,default='itag', & @@ -2140,7 +2149,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) wend = MPI_Wtime() if (mype == lead_write_task) then !** write out inline post log file - open(newunit=nolog,file='log.atm.inlinepost.f'//trim(cfhour),form='FORMATTED') + open(newunit=nolog,file=trim(fv3atm_output_dir)//'log.atm.inlinepost.f'//trim(cfhour),form='FORMATTED') write(nolog,"('completed: fv3atm')") write(nolog,"('forecast hour: ',f10.3)") nfhour write(nolog,"('valid time: ',6(i4,2x))") wrt_int_state%fdate(1:6) @@ -2313,7 +2322,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) endif else ! history bundle - filename = trim(wrtFBName)//'f'//trim(cfhour)//'.nc' + filename = trim(fv3atm_output_dir)//trim(wrtFBName)//'f'//trim(cfhour)//'.nc' endif if(mype == lead_write_task) print *,'in wrt run,filename= ',nbdl,trim(filename) @@ -2444,7 +2453,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) if (out_phase == 1 .and. mype == lead_write_task) then !** write history log file - open(newunit=nolog, file='log.atm.f'//trim(cfhour)) + open(newunit=nolog, file=trim(fv3atm_output_dir)//'log.atm.f'//trim(cfhour)) write(nolog,"('completed: fv3atm')") write(nolog,"('forecast hour: ',f10.3)") nfhour write(nolog,"('valid time: ',6(i4,2x))") wrt_int_state%fdate(1:6) diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index c0ba56dda..c07fad471 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -2,9 +2,9 @@ module post_fv3 use mpi_f08 - use module_fv3_io_def, only : wrttasks_per_group, filename_base, & - lon1, lat1, lon2, lat2, dlon, dlat, & - cen_lon, cen_lat, dxin=>dx, dyin=>dy, & + use module_fv3_io_def, only : wrttasks_per_group, fv3atm_output_dir, & + lon1, lat1, lon2, lat2, dlon, dlat, & + cen_lon, cen_lat, dxin=>dx, dyin=>dy, & stdlat1, stdlat2, output_grid use write_internal_state, only : wrt_internal_state @@ -199,6 +199,8 @@ subroutine post_run_fv3(wrt_int_state,grid_id,mype,mpicomp,lead_write, & if (grid_id > 1) then write(post_fname, '(A,I2.2)') trim(post_fname)//".nest", grid_id endif + post_fname = trim(fv3atm_output_dir)//trim(post_fname) + if (mype==0) print *,'post_fname=',trim(post_fname) call process(kth,kpv,th(1:kth),pv(1:kpv),iostatusD3D) @@ -996,13 +998,13 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) endif ! Maximum hail diameter (mm) since last output - if(trim(fieldname)=='hailcast_dhail') then + if(trim(fieldname)=='hailcast_dhail') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,hail_maxhailcast,arrayr42d,fillValue,spval) - do j=jsta,jend + do j=jsta,jend do i=ista, iend hail_maxhailcast(i,j)=arrayr42d(i,j) if(abs(arrayr42d(i,j)-fillValue) < small) hail_maxhailcast(i,j)=spval - enddo + enddo enddo endif @@ -1182,7 +1184,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - !Accumulated snowfall + !Accumulated snowfall if(trim(fieldname)=='tsnowp') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,snow_acm,arrayr42d,sm,fillValue) do j=jsta,jend @@ -1193,7 +1195,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - !Snowfall bucket + !Snowfall bucket if(trim(fieldname)=='tsnowpb') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,snow_bkt,arrayr42d,sm,fillValue) do j=jsta,jend @@ -1204,7 +1206,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - !Accumulated graupel + !Accumulated graupel if(trim(fieldname)=='frozr') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,acgraup,arrayr42d,sm,fillValue) do j=jsta,jend @@ -1215,7 +1217,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - !Graupel bucket + !Graupel bucket if(trim(fieldname)=='frozrb') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,graup_bucket,arrayr42d,sm,fillValue) do j=jsta,jend @@ -1226,7 +1228,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - !Accumulated freezing rain + !Accumulated freezing rain if(trim(fieldname)=='frzr') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,acfrain,arrayr42d,sm,fillValue) do j=jsta,jend @@ -1265,7 +1267,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) do j=jsta,jend do i=ista, iend graupelnc(i,j) = arrayr42d(i,j) - if (abs(arrayr42d(i,j)-fillValue) < small) graupelnc(i,j) = spval + if (abs(arrayr42d(i,j)-fillValue) < small) graupelnc(i,j) = spval enddo enddo endif @@ -1589,7 +1591,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) sllevel(7) = 1.0 sllevel(8) = 1.6 sllevel(9) = 3.0 - endif + endif ! liquid volumetric soil mpisture in fraction if(trim(fieldname)=='soill1') then @@ -2340,7 +2342,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) endif !FV3R if(rdaod) then - ! MERRA2 aerosols + ! MERRA2 aerosols if(trim(fieldname)=='aod550') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,aod550,arrayr42d,fillValue) do j=jsta,jend @@ -2370,7 +2372,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo enddo endif - + if(trim(fieldname)=='su_aod550') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,su_aod550,arrayr42d,fillValue) do j=jsta,jend @@ -2434,24 +2436,24 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo enddo endif - enddo + enddo do K = 1, nbin_du if ( K == 1) VarName='dust1dp' if ( K == 2) VarName='dust2dp' - if ( K == 3) VarName='dust3dp' + if ( K == 3) VarName='dust3dp' if ( K == 4) VarName='dust4dp' - if ( K == 5) VarName='dust5dp' - + if ( K == 5) VarName='dust5dp' + if(trim(fieldname)==VarName) then !$omp parallel do default(none) private(i,j,K) shared(jsta,jend,ista,iend,spval,dudp,arrayr42d,fillvalue) do j=jsta,jend do i=ista, iend dudp(i,j,K) = arrayr42d(i,j) if( abs(arrayr42d(i,j)-fillValue) < small) dudp(i,j,K) = spval - enddo - enddo - endif + enddo + enddo + endif enddo do K = 1, nbin_du @@ -2478,16 +2480,16 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) if ( K == 3) VarName='dust3wtc' if ( K == 4) VarName='dust4wtc' if ( K == 5) VarName='dust5wtc' - + if(trim(fieldname)==VarName) then !$omp parallel do default(none) private(i,j,K) shared(jsta,jend,ista,iend,spval,dusv,arrayr42d,fillvalue) do j=jsta,jend do i=ista, iend dusv(i,j,K) = arrayr42d(i,j) if( abs(arrayr42d(i,j)-fillValue) < small) dusv(i,j,K) = spval - enddo - enddo - endif + enddo + enddo + endif enddo do K = 1, nbin_ss @@ -4151,7 +4153,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - else if (nasa_on) then + else if (nasa_on) then if(trim(fieldname)=='so4') then !$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,suso,arrayr43d,fillvalue,spval) do l=1,lm @@ -4382,7 +4384,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) ! print *,'in post_gfs,ths=',maxval(ths(1:im,jsta:jend)), & ! minval(ths(1:im,jsta:jend)) -! compute cwm and max qrain in the column to be used later in precip type computation +! compute cwm and max qrain in the column to be used later in precip type computation do j=jsta,jend do i=ista,iend qrmax(i,j)=0. @@ -4469,7 +4471,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) do j=jsta,jend do i=ista, iend if(snacc_land(i,j) Date: Wed, 15 May 2024 16:27:27 -0400 Subject: [PATCH 07/10] Change compiling flag for hydrostatic options (#828) * Remove -DUSE_COND and -DMOIST_CAPPA compiling flags for hydrostatic runs; Fix the issue to write out omga and geopotential height for inline post * Add the conditions to calculate omega in post * Update CMakeLists.txt * revise conditions to calculate zint --- CMakeLists.txt | 9 +++++++- atmos_cubed_sphere | 2 +- ccpp/CMakeLists.txt | 6 +++-- io/post_fv3.F90 | 53 +++++++++++++++++++++++++++++++-------------- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c310110ca..cd8eef7e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,9 @@ set(USE_GFSL63 ON) if(MOVING_NEST) set(MOVING_NEST ON) endif() +if(HYDRO) + set(HYDRO ON) +endif() add_subdirectory(atmos_cubed_sphere) target_compile_definitions(fv3 PRIVATE BYPASS_BREED_SLP_INLINE) @@ -61,7 +64,11 @@ if(NOT PARALLEL_NETCDF) endif() if(MOVING_NEST) - list(APPEND _fv3atm_defs_private MOVING_NEST MOIST_CAPPA USE_COND) + list(APPEND _fv3atm_defs_private MOVING_NEST) + if(NOT HYDRO) + list(APPEND _fv3atm_defs_private MOIST_CAPPA USE_COND) + endif() + if(DEBUG) list(APPEND _fv3atm_defs_private DEBUG) endif() diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 97114888f..cc8afe64b 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 97114888f529c2df70ed4a350fd2428df74839ff +Subproject commit cc8afe64b9420dd7379e47cee5e3f30f4ac44b58 diff --git a/ccpp/CMakeLists.txt b/ccpp/CMakeLists.txt index ee2b9c8b2..e9a5d56c7 100644 --- a/ccpp/CMakeLists.txt +++ b/ccpp/CMakeLists.txt @@ -84,8 +84,10 @@ endif(CCPP_32BIT) #------------------------------------------------------------------------------ # Add model-specific flags for C/C++/Fortran preprocessor -add_definitions(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM) -add_definitions(-DINTERNAL_FILE_NML) +if(NOT HYDRO) + add_definitions(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM) +endif() +add_definitions(-DINTERNAL_FILE_NML -DNEMS_GSM) if(MULTI_GASES) add_definitions(-DMULTI_GASES) diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index c07fad471..1a679f18d 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -3564,6 +3564,19 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif + ! model level omga + if(trim(fieldname)=='omga') then + !$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,omga,arrayr43d,fillvalue,spval) + do l=1,lm + do j=jsta,jend + do i=ista, iend + omga(i,j,l) = arrayr43d(i,j,l) + if(abs(arrayr43d(i,j,l)-fillvalue) delp Interstitial%delz => delz @@ -1617,6 +1646,7 @@ subroutine gfdl_interstitial_create (Interstitial, is, ie, isd, ied, js, je, jsd if (do_qs) Interstitial%qs => qs if (do_qg) Interstitial%qg => qg if (do_qa) Interstitial%qc => qc + ! #ifdef USE_COND Interstitial%npzq_con = npz #else diff --git a/ccpp/data/CCPP_typedefs.meta b/ccpp/data/CCPP_typedefs.meta index 37201eb08..d1a06abf0 100644 --- a/ccpp/data/CCPP_typedefs.meta +++ b/ccpp/data/CCPP_typedefs.meta @@ -344,7 +344,7 @@ dimensions = (horizontal_loop_extent,4) type = real kind = kind_phys - active = (control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 33) + active = (control_for_drag_suite_gravity_wave_drag == 2 .or. control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 22 .or. control_for_drag_suite_gravity_wave_drag == 33) [cmm_water] standard_name = surface_drag_wind_speed_for_momentum_in_air_over_water long_name = momentum exchange coefficient over water @@ -1526,7 +1526,7 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 33) + active = (control_for_drag_suite_gravity_wave_drag == 2 .or. control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 22 .or. control_for_drag_suite_gravity_wave_drag == 33) [oa4ss] standard_name = asymmetry_of_subgrid_orography_small_scale long_name = asymmetry of subgrid height_above_mean_sea_level small scale @@ -1534,7 +1534,7 @@ dimensions = (horizontal_loop_extent,4) type = real kind = kind_phys - active = (control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 33) + active = (control_for_drag_suite_gravity_wave_drag == 2 .or. control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 22 .or. control_for_drag_suite_gravity_wave_drag == 33) [oc] standard_name = convexity_of_subgrid_orography long_name = convexity of subgrid height_above_mean_sea_level @@ -1549,7 +1549,7 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 33) + active = (control_for_drag_suite_gravity_wave_drag == 2 .or. control_for_drag_suite_gravity_wave_drag == 3 .or. control_for_drag_suite_gravity_wave_drag == 22 .or. control_for_drag_suite_gravity_wave_drag == 33) [olyr] standard_name = ozone_concentration_at_layer_for_radiation long_name = ozone concentration layer @@ -2964,6 +2964,30 @@ units = count dimensions = () type = integer +[isc1] + standard_name = starting_x_direction_index_alloc1 + long_name = starting X direction index for allocation + units = count + dimensions = () + type = integer +[iec1] + standard_name = ending_x_direction_index_alloc1 + long_name = ending X direction index for allocation + units = count + dimensions = () + type = integer +[isc2] + standard_name = starting_x_direction_index_alloc2 + long_name = starting X direction index for allocation + units = count + dimensions = () + type = integer +[iec2] + standard_name = ending_x_direction_index_alloc2 + long_name = ending X direction index for allocation + units = count + dimensions = () + type = integer [js] standard_name = starting_y_direction_index long_name = starting Y direction index @@ -2988,25 +3012,49 @@ units = count dimensions = () type = integer +[jsc1] + standard_name = starting_y_direction_index_alloc1 + long_name = starting X direction index for allocation + units = count + dimensions = () + type = integer +[jec1] + standard_name = ending_y_direction_index_alloc1 + long_name = ending X direction index for allocation + units = count + dimensions = () + type = integer +[jsc2] + standard_name = starting_y_direction_index_alloc2 + long_name = starting X direction index for allocation + units = count + dimensions = () + type = integer +[jec2] + standard_name = ending_y_direction_index_alloc2 + long_name = ending X direction index for allocation + units = count + dimensions = () + type = integer [delp] standard_name = pressure_thickness_at_Lagrangian_surface long_name = pressure thickness at Lagrangian surface units = Pa - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [delz] standard_name = thickness_at_Lagrangian_surface long_name = thickness at Lagrangian_surface units = m - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_thickness_at_Lagrangian_surface) + dimensions = (starting_x_direction_index_alloc2:ending_x_direction_index_alloc2,starting_y_direction_index_alloc2:ending_y_direction_index_alloc2,1:vertical_dimension_for_thickness_at_Lagrangian_surface) type = real kind = kind_dyn [area] standard_name = cell_area_for_fast_physics long_name = area of the grid cell for fast physics units = m2 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1) type = real kind = kind_grid [ng] @@ -3049,35 +3097,35 @@ standard_name = log_pressure_at_Lagrangian_surface long_name = logarithm of pressure at Lagrangian surface units = Pa - dimensions = (starting_x_direction_index:ending_x_direction_index,1:vertical_dimension_for_fast_physics_plus_one,starting_y_direction_index:ending_y_direction_index) + dimensions = (starting_x_direction_index_alloc2:ending_x_direction_index_alloc2,1:vertical_dimension_for_fast_physics_plus_one,starting_y_direction_index_alloc2:ending_y_direction_index_alloc2) type = real kind = kind_dyn [phis] standard_name = surface_geopotential_at_Lagrangian_surface long_name = surface geopotential at Lagrangian surface units = m2 s-2 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1) type = real kind = kind_dyn [pkz] standard_name = finite_volume_mean_edge_pressure_raised_to_the_power_of_kappa long_name = finite-volume mean edge pressure raised to the power of kappa units = 1 - dimensions = (starting_x_direction_index:ending_x_direction_index,starting_y_direction_index:ending_y_direction_index,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc2:ending_x_direction_index_alloc2,starting_y_direction_index_alloc2:ending_y_direction_index_alloc2,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [pt] standard_name = virtual_temperature_at_Lagrangian_surface long_name = virtual temperature at Lagrangian surface units = K - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [qvi] standard_name = gas_tracers_for_multi_gas_physics_at_Lagrangian_surface long_name = gas tracers for multi gas physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics,0:number_of_gases_for_multi_gases_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics,1:number_of_gases_for_multi_gases_physics) type = real kind = kind_dyn active = (number_of_gases_for_multi_gases_physics > 0) @@ -3085,56 +3133,56 @@ standard_name = water_vapor_specific_humidity_at_Lagrangian_surface long_name = water vapor specific humidity updated by fast physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [ql] standard_name = cloud_liquid_water_specific_humidity_at_Lagrangian_surface long_name = cloud liquid water specific humidity updated by fast physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [qi] standard_name = cloud_ice_specific_humidity_at_Lagrangian_surface long_name = cloud ice specific humidity updated by fast physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [qr] standard_name = cloud_rain_specific_humidity_at_Lagrangian_surface long_name = cloud rain specific humidity updated by fast physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [qs] standard_name = cloud_snow_specific_humidity_at_Lagrangian_surface long_name = cloud snow specific humidity updated by fast physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [qg] standard_name = cloud_graupel_specific_humidity_at_Lagrangian_surface long_name = cloud graupel specific humidity updated by fast physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [qc] standard_name = cloud_fraction_at_Lagrangian_surface long_name = cloud fraction at Lagrangian surface units = none - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_fast_physics) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_fast_physics) type = real kind = kind_dyn [q_con] standard_name = cloud_condensed_water_specific_humidity_at_Lagrangian_surface long_name = cloud condensed water specific humidity updated by fast physics at Lagrangian surface units = kg kg-1 - dimensions = (starting_x_direction_index_domain:ending_x_direction_index_domain,starting_y_direction_index_domain:ending_y_direction_index_domain,1:vertical_dimension_for_condensed_water_at_Lagrangian_surface) + dimensions = (starting_x_direction_index_alloc1:ending_x_direction_index_alloc1,starting_y_direction_index_alloc1:ending_y_direction_index_alloc1,1:vertical_dimension_for_condensed_water_at_Lagrangian_surface) type = real kind = kind_dyn [nthreads] diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index a15b053d0..31f48bf5d 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -2746,12 +2746,13 @@ subroutine sfcprop_create (Sfcprop, IM, Model) ! end if - allocate (Sfcprop%rmol (IM )) - allocate (Sfcprop%flhc (IM )) - allocate (Sfcprop%flqc (IM )) - Sfcprop%rmol = clear_val - Sfcprop%flhc = clear_val - Sfcprop%flqc = clear_val + allocate (Sfcprop%rmol (IM )) + allocate (Sfcprop%flhc (IM )) + allocate (Sfcprop%flqc (IM )) + Sfcprop%rmol = clear_val + Sfcprop%flhc = clear_val + Sfcprop%flqc = clear_val + if (Model%do_mynnsfclay) then ! For MYNN surface layer scheme !print*,"Allocating all MYNN-sfclay variables" diff --git a/ccpp/data/GFS_typedefs.meta b/ccpp/data/GFS_typedefs.meta index 0986bdf41..5ac00e403 100644 --- a/ccpp/data/GFS_typedefs.meta +++ b/ccpp/data/GFS_typedefs.meta @@ -992,7 +992,6 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (control_for_land_surface_scheme == identifier_for_ruc_land_surface_scheme) [fire_heat_flux] standard_name = surface_fire_heat_flux long_name = heat flux of fire at the surface @@ -1179,7 +1178,7 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (control_for_deep_convection_scheme == identifier_for_grell_freitas_deep_convection) + active = (control_for_deep_convection_scheme == identifier_for_grell_freitas_deep_convection .or. control_for_deep_convection_scheme == identifier_for_c3_deep_convection) [conv_act] standard_name = consecutive_calls_for_grell_freitas_convection long_name = Memory counter for GF @@ -1943,7 +1942,6 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme) [flhc] standard_name = surface_exchange_coefficient_for_heat long_name = surface exchange coefficient for heat @@ -1951,7 +1949,6 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme) [flqc] standard_name = surface_exchange_coefficient_for_moisture long_name = surface exchange coefficient for moisture @@ -1959,7 +1956,6 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme) [chs2] standard_name = surface_exchange_coefficient_for_heat_at_2m long_name = exchange coefficient for heat at 2 meters @@ -2763,7 +2759,7 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (flag_for_surface_flux_coupling .or. flag_for_ocean_wave_coupling) + active = (flag_for_surface_flux_coupling .or. flag_for_chemistry_coupling .or. flag_for_ocean_wave_coupling) [v10mi_cpl] standard_name = y_wind_at_10m_for_coupling long_name = instantaneous V10m @@ -2771,7 +2767,7 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (flag_for_surface_flux_coupling .or. flag_for_ocean_wave_coupling) + active = (flag_for_surface_flux_coupling .or. flag_for_chemistry_coupling .or. flag_for_ocean_wave_coupling) [tsfci_cpl] standard_name = surface_skin_temperature_for_coupling long_name = instantaneous sfc temperature @@ -2779,7 +2775,7 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys - active = (flag_for_surface_flux_coupling) + active = (flag_for_surface_flux_coupling .or. flag_for_chemistry_coupling .or. flag_for_land_coupling) [psurfi_cpl] standard_name = surface_air_pressure_for_coupling long_name = instantaneous sfc pressure @@ -3379,7 +3375,6 @@ dimensions = (number_of_lines_in_internal_namelist) type = character kind = len=256 - active = (number_of_lines_in_internal_namelist > 0) [logunit] standard_name = iounit_of_log long_name = fortran unit number for logfile @@ -6721,14 +6716,12 @@ units = flag dimensions = () type = logical - active = (do_smoke_coupling) [enh_mix] standard_name = do_planetary_boundary_layer_fire_enhancement long_name = flag for rrfs smoke mynn enh vermix units = flag dimensions = () type = logical - active = (do_smoke_coupling) [smoke_dir_fdb_coef] standard_name = smoke_dust_direct_fdb_coef long_name = smoke dust direct feedback coefficents @@ -6736,7 +6729,6 @@ dimensions = (7) type = real kind = kind_phys - active = (do_smoke_coupling) [smoke_conv_wet_coef] standard_name = smoke_dust_conv_wet_coef long_name = smoke dust convetive wet scavanging coefficents @@ -6744,7 +6736,6 @@ dimensions = (3) type = real kind = kind_phys - active = (do_smoke_coupling) [dust_moist_correction] standard_name = dust_moist_correction_fengsha_dust_scheme long_name = moisture correction term for fengsha dust emission @@ -6752,7 +6743,6 @@ dimensions = () type = real kind = kind_phys - active = (do_smoke_coupling) [dust_drylimit_factor] standard_name = dust_drylimit_factor_fengsha_dust_scheme long_name = moisture correction term for drylimit in fengsha dust emission @@ -6760,14 +6750,12 @@ dimensions = () type = real kind = kind_phys - active = (do_smoke_coupling) [dust_moist_opt] standard_name = control_for_dust_soil_moisture_option long_name = smoke dust moisture parameterization 1 - fecan 2 - shao units = index dimensions = () type = integer - active = (do_smoke_coupling) [dust_alpha] standard_name = alpha_fengsha_dust_scheme long_name = alpha paramter for fengsha dust scheme @@ -6775,7 +6763,6 @@ dimensions = () type = real kind = kind_phys - active = (do_smoke_coupling) [dust_gamma] standard_name = gamma_fengsha_dust_scheme long_name = gamma paramter for fengsha dust scheme @@ -6783,7 +6770,6 @@ dimensions = () type = real kind = kind_phys - active = (do_smoke_coupling) [wetdep_ls_alpha] standard_name = alpha_for_ls_wet_depoistion long_name = alpha paramter for ls wet deposition @@ -6791,126 +6777,108 @@ dimensions = () type = real kind = kind_phys - active = (do_smoke_coupling) [ebb_dcycle] standard_name = control_for_diurnal_cycle_of_biomass_burning_emissions long_name = rrfs smoke diurnal cycle option units = index dimensions = () type = integer - active = (do_smoke_coupling) [seas_opt] standard_name = control_for_smoke_sea_salt long_name = rrfs smoke sea salt emission option units = index dimensions = () type = integer - active = (do_smoke_coupling) [dust_opt] standard_name = control_for_smoke_dust long_name = rrfs smoke dust chem option units = index dimensions = () type = integer - active = (do_smoke_coupling) [drydep_opt] standard_name = control_for_smoke_dry_deposition long_name = rrfs smoke dry deposition option units = index dimensions = () type = integer - active = (do_smoke_coupling) [coarsepm_settling] standard_name = control_for_smoke_pm_settling long_name = rrfs smoke coarsepm settling option units = index dimensions = () type = integer - active = (do_smoke_coupling) [plume_wind_eff] standard_name = option_for_wind_effects_on_smoke_plumerise long_name = wind effect plumerise option units = index dimensions = () type = integer - active = (do_smoke_coupling) [extended_sd_diags] standard_name = flag_for_extended_smoke_dust_diagnostics long_name = flag for extended smoke dust diagnostics units = flag dimensions = () type = logical - active = (do_smoke_coupling) [wetdep_ls_opt] standard_name = control_for_smoke_wet_deposition long_name = rrfs smoke large scale wet deposition option units = index dimensions = () type = integer - active = (do_smoke_coupling) [do_plumerise] standard_name = do_smoke_plumerise long_name = rrfs smoke plumerise option units = index dimensions = () type = logical - active = (do_smoke_coupling) [plumerisefire_frq] standard_name = smoke_plumerise_frequency long_name = rrfs smoke add smoke option units = min dimensions = () type = integer - active = (do_smoke_coupling) [n_dbg_lines] standard_name = smoke_debug_lines long_name = rrfs smoke add smoke option units = index dimensions = () type = integer - active = (do_smoke_coupling) [addsmoke_flag] standard_name = control_for_smoke_biomass_burning_emissions long_name = rrfs smoke add smoke option units = index dimensions = () type = integer - active = (do_smoke_coupling) [smoke_forecast] standard_name = do_smoke_forecast long_name = index for rrfs smoke forecast units = index dimensions = () type = integer - active = (do_smoke_coupling) [aero_ind_fdb] standard_name = do_smoke_aerosol_indirect_feedback long_name = flag for wfa ifa emission indirect feedback units = flag dimensions = () type = logical - active = (do_smoke_coupling) [aero_dir_fdb] standard_name = do_smoke_aerosol_direct_feedback long_name = flag for smoke and dust radiation feedback units = flag dimensions = () type = logical - active = (do_smoke_coupling) [rrfs_smoke_debug] standard_name = do_smoke_debug long_name = flag for rrfs smoke plumerise debug units = flag dimensions = () type = logical - active = (do_smoke_coupling) [do_smoke_transport] standard_name = do_smoke_conv_transport long_name = flag for rrfs smoke convective transport units = flag dimensions = () type = logical - active = (do_smoke_coupling) [ncnvcld3d] standard_name = number_of_convective_cloud_variables_in_xyz_dimensioned_restart_array long_name = number of convective 3d clouds fields diff --git a/ccpp/driver/CCPP_driver.F90 b/ccpp/driver/CCPP_driver.F90 index ec36e2ad3..8e4c019b3 100644 --- a/ccpp/driver/CCPP_driver.F90 +++ b/ccpp/driver/CCPP_driver.F90 @@ -85,6 +85,7 @@ subroutine CCPP_step (step, nblks, ierr) ! number are not used; set to safe values cdata_domain%blk_no = 1 cdata_domain%thrd_no = 1 + cdata_domain%thrd_cnt = 1 ! Allocate cdata structures for blocks and threads if (.not.allocated(cdata_block)) allocate(cdata_block(1:nblks,1:nthrdsX)) @@ -95,6 +96,7 @@ subroutine CCPP_step (step, nblks, ierr) ! Assign the correct block and thread numbers cdata_block(nb,nt)%blk_no = nb cdata_block(nb,nt)%thrd_no = nt + cdata_block(nb,nt)%thrd_cnt = nthrdsX end do end do @@ -166,9 +168,11 @@ subroutine CCPP_step (step, nblks, ierr) ! because threads are used on the host model side for blocking GFS_control%nthreads = 1 -!$OMP parallel num_threads (nthrds) & -!$OMP default (shared) & -!$OMP private (nb,nt,ntX,ierr2) & +!$OMP parallel num_threads (nthrds) & +!$OMP default (none) & +!$OMP shared (nblks, nthrdsX, non_uniform_blocks, & +!$OMP cdata_block,ccpp_suite, step) & +!$OMP private (nb,nt,ntX,ierr2) & !$OMP reduction (+:ierr) #ifdef _OPENMP nt = omp_get_thread_num()+1 diff --git a/ccpp/framework b/ccpp/framework index 741212e49..ccfefcd0b 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit 741212e4962d371520f773e2be9481142f79635e +Subproject commit ccfefcd0b426e011f94137031d5f7c2a4dda2659 diff --git a/ccpp/physics b/ccpp/physics index 89ddce75d..16a1d8817 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 89ddce75d4d64252f1871c095a362ad391065b89 +Subproject commit 16a1d881774d795f46db16017aeed7fc351d661a diff --git a/stochastic_physics/stochastic_physics_wrapper.F90 b/stochastic_physics/stochastic_physics_wrapper.F90 index b76c52a39..842495569 100644 --- a/stochastic_physics/stochastic_physics_wrapper.F90 +++ b/stochastic_physics/stochastic_physics_wrapper.F90 @@ -282,6 +282,7 @@ subroutine stochastic_physics_wrapper (GFS_Control, GFS_Data, Atm_block, ierr) endif select case (trim(GFS_Control%lndp_var_list(v))) + ! DH* is this correct? shouldn't this be slc ? case('smc') ! stype used to fetch soil params stype(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Sfcprop%stype(1:GFS_Control%blksz(nb)) From b3640007c51c8899533e1b835940a2030a768fcc Mon Sep 17 00:00:00 2001 From: Dusan Jovic <48258889+DusanJovic-NOAA@users.noreply.github.com> Date: Tue, 28 May 2024 15:32:19 -0400 Subject: [PATCH 10/10] Bucket change during integration (#835) * set up fhour with multiple values * updates for diag time * add fixes for two bucket * Update fhzero global attribute at runtime * Update to speed uo writing restart files * allow fhzero to be non-integer * Move ESMF_InfoBroadcast call outside ESMF_VMEpochEnter/ESMF_VMEpochExit section to avoid hanging --------- Co-authored-by: Jun.Wang Co-authored-by: Dusan Javic --- atmos_model.F90 | 72 +++++- ccpp/data/GFS_typedefs.F90 | 23 +- fv3_cap.F90 | 17 ++ io/fv3atm_history_io.F90 | 5 +- io/module_write_internal_state.F90 | 2 +- io/module_wrt_grid_comp.F90 | 338 +++++++++++++++-------------- io/post_fv3.F90 | 9 +- module_fcst_grid_comp.F90 | 38 ++++ 8 files changed, 316 insertions(+), 188 deletions(-) diff --git a/atmos_model.F90 b/atmos_model.F90 index a2ca32a75..92fa59742 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -533,12 +533,12 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) type (time_type), intent(in) :: Time_init, Time, Time_step !--- local variables --- integer :: unit, i - integer :: mlon, mlat, nlon, nlat, nlev, sec + integer :: mlon, mlat, nlon, nlat, nlev, sec, sec_lastfhzerofh integer :: ierr, io, logunit integer :: tile_num integer :: isc, iec, jsc, jec real(kind=GFS_kind_phys) :: dt_phys - logical :: p_hydro, hydro + logical :: p_hydro, hydro, tmpflag_fhzero logical, save :: block_message = .true. type(GFS_init_type) :: Init_parm integer :: bdat(8), cdat(8) @@ -789,8 +789,33 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) !--- WARNING: For special cases that model needs to restart at non-multiple of fhzero !--- the fields in first output files are not accumulated from the beginning of !--- the bucket, but the restart time. - if (mod(sec,int(GFS_Control%fhzero*3600.)) /= 0) then - diag_time = Time - real_to_time_type(mod(int((GFS_Control%kdt - 1)*dt_phys/3600.),int(GFS_Control%fhzero))*3600.0) + if( GFS_Control%fhzero_array(1) > 0. ) then + fhzero_loop: do i=1,size(GFS_Control%fhzero_array) + tmpflag_fhzero= .false. + if( GFS_Control%fhzero_array(i) > 0.) then + if( i == 1 ) then + if( sec <= GFS_Control%fhzero_fhour(i)*3600. ) tmpflag_fhzero = .true. + else if( i > 1 ) then + if( sec > GFS_Control%fhzero_fhour(i-1)*3600. .and. sec <=GFS_Control%fhzero_fhour(i)*3600. ) & + tmpflag_fhzero = .true. + endif + if( tmpflag_fhzero ) then + GFS_Control%fhzero = GFS_Control%fhzero_array(i) + if( GFS_Control%fhzero > 0) then + sec_lastfhzerofh = (int(sec/3600.)/int(GFS_Control%fhzero))*int(GFS_Control%fhzero)*3600 + else + sec_lastfhzerofh = 0 + endif + endif + endif + enddo fhzero_loop + else + sec_lastfhzerofh = 0 + endif + if (mpp_pe() == mpp_root_pe()) print *,'in atmos_model, fhzero=',GFS_Control%fhzero, 'fhour=',sec/3600.,sec_lastfhzerofh/3600 + + if (mod((sec-sec_lastfhzerofh),int(GFS_Control%fhzero*3600.)) /= 0) then + diag_time = Time - real_to_time_type(mod(int((GFS_Control%kdt - 1)*dt_phys-sec_lastfhzerofh),int(GFS_Control%fhzero))*3600.0) if (mpp_pe() == mpp_root_pe()) print *,'Warning: in atmos_init,start at non multiple of fhzero' endif if (Atmos%iau_offset > zero) then @@ -949,8 +974,9 @@ subroutine update_atmos_model_state (Atmos, rc) type (atmos_data_type), intent(inout) :: Atmos integer, optional, intent(out) :: rc !--- local variables - integer :: localrc + integer :: i, localrc, sec_lastfhzerofh integer :: isec, seconds, isec_fhzero + logical :: tmpflag_fhzero real(kind=GFS_kind_phys) :: time_int, time_intfull ! if (present(rc)) rc = ESMF_SUCCESS @@ -1001,16 +1027,38 @@ subroutine update_atmos_model_state (Atmos, rc) GFS_control%levs, 1, 1, 1.0_GFS_kind_phys, time_int, time_intfull, & GFS_control%fhswr, GFS_control%fhlwr) endif - if (nint(GFS_control%fhzero) > 0) then - if (mod(isec,3600*nint(GFS_control%fhzero)) == 0) diag_time = Atmos%Time + + !--- find current fhzero + if( GFS_Control%fhzero_array(1) > 0. ) then + fhzero_loop: do i=1,size(GFS_Control%fhzero_array) + tmpflag_fhzero = .false. + if( GFS_Control%fhzero_array(i) > 0.) then + if( i == 1 ) then + if( seconds <= GFS_Control%fhzero_fhour(i)*3600. ) tmpflag_fhzero = .true. + else if( i > 1 ) then + if( seconds > GFS_Control%fhzero_fhour(i-1)*3600. .and. seconds <= GFS_Control%fhzero_fhour(i)*3600. ) & + tmpflag_fhzero = .true. + endif + if( tmpflag_fhzero) then + GFS_Control%fhzero = GFS_Control%fhzero_array(i) + if( GFS_Control%fhzero > 0) then + sec_lastfhzerofh = (int(seconds/3600.)/int(GFS_Control%fhzero))*int(GFS_Control%fhzero)*3600 + else + sec_lastfhzerofh = 0 + endif + endif + endif + enddo fhzero_loop else - if (mod(isec,nint(3600*GFS_control%fhzero)) == 0) diag_time = Atmos%Time + sec_lastfhzerofh = 0 endif - call diag_send_complete_instant (Atmos%Time) + if (mpp_pe() == mpp_root_pe()) print *,'in atmos_model update, fhzero=',GFS_Control%fhzero, 'fhour=',seconds/3600.,sec_lastfhzerofh/3600. - - !--- this may not be necessary once write_component is fully implemented - !!!call diag_send_complete_extra (Atmos%Time) + if (nint(GFS_Control%fhzero) > 0) then + if (mod(isec - sec_lastfhzerofh,nint(GFS_Control%fhzero*3600.)) == 0) diag_time = Atmos%Time +! if (mpp_pe() == mpp_root_pe()) print *,'in atmos_model update time=',isec/3600.,'last fhzeo=',sec_lastfhzerofh + endif + call diag_send_complete_instant (Atmos%Time) !--- get bottom layer data from dynamical core for coupling call atmosphere_get_bottom_layer (Atm_block, DYCORE_Data) diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index 31f48bf5d..40c33cfc5 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -710,7 +710,9 @@ module GFS_typedefs !< for use with internal file reads integer :: input_nml_file_length !< length (number of lines) in namelist for internal reads integer :: logunit - real(kind=kind_phys) :: fhzero !< hours between clearing of diagnostic buckets + real(kind=kind_phys) :: fhzero !< hours between clearing of diagnostic buckets (current bucket) + real(kind=kind_phys) :: fhzero_array(2) !< array to hold the the hours between clearing of diagnostic buckets + real(kind=kind_phys) :: fhzero_fhour(2) !< the maximum forecast length for the hours between clearing of diagnostic buckets logical :: ldiag3d !< flag for 3d diagnostic fields logical :: qdiag3d !< flag for 3d tracer diagnostic fields logical :: flag_for_gwd_generic_tend !< true if GFS_GWD_generic should calculate tendencies @@ -3327,6 +3329,8 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & !--- BEGIN NAMELIST VARIABLES real(kind=kind_phys) :: fhzero = 0.0 !< hours between clearing of diagnostic buckets + real(kind=kind_phys) :: fhzero_array(1:2) = 0.0 !< array with hours between clearing of diagnostic buckets + real(kind=kind_phys) :: fhzero_fhour(1:2) = 0.0 !< the maximum forecast length for the hours between clearing of diagnostic buckets logical :: ldiag3d = .false. !< flag for 3d diagnostic fields logical :: qdiag3d = .false. !< flag for 3d tracer diagnostic fields logical :: lssav = .false. !< logical flag for storing diagnostics @@ -3983,9 +3987,9 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & NAMELIST /gfs_physics_nml/ & !--- general parameters - fhzero, ldiag3d, qdiag3d, lssav, naux2d, dtend_select, & - naux3d, aux2d_time_avg, aux3d_time_avg, fhcyc, & - thermodyn_id, sfcpress_id, & + fhzero, fhzero_array, fhzero_fhour, ldiag3d, qdiag3d, lssav, & + naux2d, dtend_select, naux3d, aux2d_time_avg, & + aux3d_time_avg, fhcyc, thermodyn_id, sfcpress_id, & !--- coupling parameters cplflx, cplice, cplocn2atm, cplwav, cplwav2atm, cplaqm, & cplchm, cpllnd, cpllnd2atm, cpl_imp_mrg, cpl_imp_dbg, & @@ -4196,6 +4200,11 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%fn_nml = fn_nml Model%logunit = logunit Model%fhzero = fhzero + Model%fhzero_array = fhzero_array + Model%fhzero_fhour = fhzero_fhour + if( Model%fhzero_array(1) > 0. ) then + Model%fhzero = Model%fhzero_array(1) + endif Model%ldiag3d = ldiag3d Model%qdiag3d = qdiag3d if (qdiag3d .and. .not. ldiag3d) then @@ -5621,6 +5630,10 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%restart = restart Model%lsm_cold_start = .not. restart Model%hydrostatic = hydrostatic + if (Model%me == Model%master) then + print *,'in atm phys init, phour=',Model%phour,'fhour=',Model%fhour,'zhour=',Model%zhour,'kdt=',Model%kdt + endif + if(Model%hydrostatic .and. Model%lightning_threat) then write(0,*) 'Turning off lightning threat index for hydrostatic run.' @@ -6414,6 +6427,8 @@ subroutine control_print(Model) print *, ' nlunit : ', Model%nlunit print *, ' fn_nml : ', trim(Model%fn_nml) print *, ' fhzero : ', Model%fhzero + print *, ' fhzero_array : ', Model%fhzero_array + print *, ' fhzero_fhour : ', Model%fhzero_fhour print *, ' ldiag3d : ', Model%ldiag3d print *, ' qdiag3d : ', Model%qdiag3d print *, ' lssav : ', Model%lssav diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 713460fe3..20c2bcc7a 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -1114,6 +1114,8 @@ subroutine ModelAdvance_phase2(gcomp, rc) real(kind=8) :: MPI_Wtime, timep2rs + character(len=ESMF_MAXSTR) :: fb_name + type(ESMF_Info) :: info !----------------------------------------------------------------------------- rc = ESMF_SUCCESS @@ -1182,6 +1184,21 @@ subroutine ModelAdvance_phase2(gcomp, rc) call ESMF_TraceRegionExit("ESMF_VMEpoch:fcstFB->wrtFB", rc=rc) + do j=1, FBCount + + ! Update fcstFB attributes from fcst PEs to all PEs in this VM + ! This is needed in case some attributes are updated during run time + call ESMF_FieldBundleGet(fcstFB(j), name=fb_name, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (fb_name(1:8) /= "restart_") then + call ESMF_InfoGetFromHost(fcstFB(j), info=info, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_InfoBroadcast(info, rootPet=fcstPetList(1), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + endif + + enddo + call ESMF_LogWrite('Model Advance: before wrtcomp run ', ESMF_LOGMSG_INFO, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return diff --git a/io/fv3atm_history_io.F90 b/io/fv3atm_history_io.F90 index 7171aa673..6fe537e04 100644 --- a/io/fv3atm_history_io.F90 +++ b/io/fv3atm_history_io.F90 @@ -50,7 +50,8 @@ module fv3atm_history_io_mod integer :: tot_diag_idx = 0 integer :: isco=0,ieco=0,jsco=0,jeco=0,num_axes_phys=0 - integer :: fhzero=0, ncld=0, nsoil=0, nsoil_lsm=0, imp_physics=0, landsfcmdl=0 + integer :: ncld=0, nsoil=0, nsoil_lsm=0, imp_physics=0, landsfcmdl=0 + real(4) :: fhzero=0. real(4) :: dtp=0 integer,dimension(:), pointer :: levo => null() integer,dimension(:), pointer :: nstt => null() @@ -183,7 +184,7 @@ subroutine history_type_register(hist, Diag, Time, Atm_block, Model, xlon, xlat, hist%ieco = Atm_block%iec hist%jsco = Atm_block%jsc hist%jeco = Atm_block%jec - hist%fhzero = nint(Model%fhzero) + hist%fhzero = Model%fhzero ! hist%ncld = Model%ncld hist%ncld = Model%imp_physics hist%nsoil = Model%lsoil diff --git a/io/module_write_internal_state.F90 b/io/module_write_internal_state.F90 index 51c422227..0f81fc9c0 100644 --- a/io/module_write_internal_state.F90 +++ b/io/module_write_internal_state.F90 @@ -93,7 +93,7 @@ module write_internal_state logical :: write_dopost !< True if inline post is requested. character(80) :: post_namelist !< File name of the inline post namelist. ! - integer :: fhzero !< Hours between clearing of diagnostic buckets. + real(4) :: fhzero !< Hours between clearing of diagnostic buckets. integer :: ntrac !< Number of tracers. integer :: ncld !< Number of hydrometeors. integer :: nsoil !< Number of soil layers. diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index 5a3945714..af468c57e 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -80,6 +80,7 @@ module module_wrt_grid_comp type(ESMF_FieldBundle) :: gridFB integer :: FBCount character(len=esmf_maxstr),allocatable :: fcstItemNameList(:) + character(128) :: FBlist_outfilename(100) logical :: top_parent_is_global ! !----------------------------------------------------------------------- @@ -196,7 +197,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, integer :: attCount, jidx, idx, noutfile character(19) :: newdate - character(128) :: FBlist_outfilename(100), outfile_name + character(128) :: outfile_name character(128),dimension(:,:), allocatable :: outfilename real(8), dimension(:), allocatable :: slat real(8), dimension(:), allocatable :: lat, lon @@ -215,8 +216,8 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, integer :: grid_id logical :: history_file_on_native_grid - character(len=esmf_maxstr) :: output_grid_name ! + character(ESMF_MAXSTR) :: fb_name1, fb_name2 !----------------------------------------------------------------------- !*********************************************************************** !----------------------------------------------------------------------- @@ -1161,129 +1162,6 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, enddo ! FBCount - ! add output grid related attributes, only for history files(bundles), skip restart - if (FBlist_outfilename(i)(1:8) /= 'restart_') then - - call ESMF_AttributeGet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3-nooutput", & - name="output_grid", value=output_grid_name, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - attrList=(/"source","grid "/), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="source", value="FV3GFS", rc=rc) - - if (trim(output_grid_name) == 'cubed_sphere_grid') then - - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="grid", value="cubed_sphere", rc=rc) - - else if (trim(output_grid_name) == 'gaussian_grid') then - - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="grid", value="gaussian", rc=rc) - call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - attrList=(/"im","jm"/), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="im", value=imo(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="jm", value=jmo(grid_id), rc=rc) - - else if (trim(output_grid_name) == 'regional_latlon' & - .or. trim(output_grid_name) == 'regional_latlon_moving' & - .or. trim(output_grid_name) == 'global_latlon') then - - ! for 'regional_latlon_moving' lon1/2 and lat1/2 will be overwritten in run phase - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="grid", value="latlon", rc=rc) - call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - attrList=(/"lon1","lat1","lon2","lat2","dlon","dlat"/), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlon", value=dlon(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlat", value=dlat(grid_id), rc=rc) - if (trim(output_grid_name) /= 'regional_latlon_moving') then - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon2", value=lon2(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat2", value=lat2(grid_id), rc=rc) - endif - else if (trim(output_grid_name) == 'rotated_latlon' & - .or. trim(output_grid_name) == 'rotated_latlon_moving') then - - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="grid", value="rotated_latlon", rc=rc) - call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - attrList=(/"cen_lon",& - "cen_lat",& - "lon1 ",& - "lat1 ",& - "lon2 ",& - "lat2 ",& - "dlon ",& - "dlat "/), rc=rc) - ! for 'rotated_latlon_moving' cen_lon and cen_lat will be overwritten in run phase - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lon", value=cen_lon(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lat", value=cen_lat(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlon", value=dlon(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlat", value=dlat(grid_id), rc=rc) - if (trim(output_grid_name) /= 'rotated_latlon_moving') then - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon2", value=lon2(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat2", value=lat2(grid_id), rc=rc) - endif - else if (trim(output_grid_name) == 'lambert_conformal') then - - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="grid", value="lambert_conformal", rc=rc) - call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - attrList=(/"cen_lon",& - "cen_lat",& - "stdlat1",& - "stdlat2",& - "nx ",& - "ny ",& - "lon1 ",& - "lat1 ",& - "dx ",& - "dy "/), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lon", value=cen_lon(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lat", value=cen_lat(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="stdlat1", value=stdlat1(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="stdlat2", value=stdlat2(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="nx", value=imo(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="ny", value=jmo(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dx", value=dx(grid_id), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dy", value=dy(grid_id), rc=rc) - - end if - end if - enddo ! end wrt_int_state%FBCount ! ! add time Attribute @@ -1785,7 +1663,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) type(write_wrap) :: wrap type(wrt_internal_state),pointer :: wrt_int_state ! - integer :: i,j,n,mype,nolog, grid_id, localPet + integer :: i,j,n,m, mype,nolog, grid_id, localPet ! integer :: nf_hours,nf_seconds,nf_minutes integer :: fcst_seconds @@ -1832,6 +1710,8 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) real, allocatable :: output_fh(:) logical :: is_restart_bundle, restart_written integer :: tileCount + type(ESMF_Info) :: fcstInfo, wrtInfo + character(len=ESMF_MAXSTR) :: output_grid_name ! !----------------------------------------------------------------------- !*********************************************************************** @@ -1911,6 +1791,22 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) fieldbundle=file_bundle, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + do m=1, wrt_int_state%FBCount + if (trim_regridmethod_suffix(fcstItemNameList(i)) == trim_regridmethod_suffix(FBlist_outfilename(m))) then + + call ESMF_InfoGetFromHost(file_bundle, info=fcstInfo, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_InfoGetFromHost(wrt_int_state%wrtFB(m), info=wrtInfo, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_InfoUpdate(lhs=wrtInfo, rhs=fcstInfo, recursive=.true., overwrite=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! if (lprnt) call print_att_list(wrt_int_state%wrtFB(m), rc) + + end if + end do + ! see whether a "mirror_" FieldBundle exists, i.e. dealing with moving domain that needs updated Regrid() here. call ESMF_StateGet(imp_state_write, itemName="mirror_"//trim(fcstItemNameList(i)), & itemType=itemType, rc=rc) @@ -2209,43 +2105,125 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) name="grid_id", value=grid_id, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! update lon1/2 and lat1/2 for regional_latlon_moving - if (trim(output_grid(grid_id)) == 'regional_latlon_moving') then - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lon2", value=lon2(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lat2", value=lat2(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (wrtFBName(1:18) == 'cubed_sphere_grid_') then + output_grid_name = "cubed_sphere_grid" + else + output_grid_name = output_grid(grid_id) endif - ! update cen_lon/cen_lat, lon1/2 and lat1/2 for rotated_latlon_moving - if (trim(output_grid(grid_id)) == 'rotated_latlon_moving') then - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="cen_lon", value=cen_lon(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="cen_lat", value=cen_lat(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lon2", value=lon2(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! add output grid related attributes, only for history files(bundles), skip restart + if (.not.is_restart_bundle) then + + call ESMF_AttributeAdd(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + attrList=(/"source","grid "/), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & - name="lat2", value=lat2(grid_id), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - endif + name="source", value="FV3GFS", rc=rc) + + if (trim(output_grid_name) == 'cubed_sphere_grid') then + + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="grid", value="cubed_sphere", rc=rc) + + else if (trim(output_grid_name) == 'gaussian_grid') then + + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="grid", value="gaussian", rc=rc) + call ESMF_AttributeAdd(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + attrList=(/"im","jm"/), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="im", value=imo(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="jm", value=jmo(grid_id), rc=rc) + + else if (trim(output_grid_name) == 'regional_latlon' & + .or. trim(output_grid_name) == 'regional_latlon_moving' & + .or. trim(output_grid_name) == 'global_latlon') then + + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="grid", value="latlon", rc=rc) + call ESMF_AttributeAdd(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + attrList=(/"lon1","lat1","lon2","lat2","dlon","dlat"/), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="dlon", value=dlon(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="dlat", value=dlat(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lon1", value=lon1(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lat1", value=lat1(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lon2", value=lon2(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lat2", value=lat2(grid_id), rc=rc) + else if (trim(output_grid_name) == 'rotated_latlon' & + .or. trim(output_grid_name) == 'rotated_latlon_moving') then + + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="grid", value="rotated_latlon", rc=rc) + call ESMF_AttributeAdd(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + attrList=(/"cen_lon",& + "cen_lat",& + "lon1 ",& + "lat1 ",& + "lon2 ",& + "lat2 ",& + "dlon ",& + "dlat "/), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="cen_lon", value=cen_lon(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="cen_lat", value=cen_lat(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="dlon", value=dlon(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="dlat", value=dlat(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lon1", value=lon1(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lat1", value=lat1(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lon2", value=lon2(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lat2", value=lat2(grid_id), rc=rc) + else if (trim(output_grid_name) == 'lambert_conformal') then + + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="grid", value="lambert_conformal", rc=rc) + call ESMF_AttributeAdd(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + attrList=(/"cen_lon",& + "cen_lat",& + "stdlat1",& + "stdlat2",& + "nx ",& + "ny ",& + "lon1 ",& + "lat1 ",& + "dx ",& + "dy "/), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="cen_lon", value=cen_lon(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="cen_lat", value=cen_lat(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="stdlat1", value=stdlat1(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="stdlat2", value=stdlat2(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="nx", value=imo(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="ny", value=jmo(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lat1", value=lat1(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="lon1", value=lon1(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="dx", value=dx(grid_id), rc=rc) + call ESMF_AttributeSet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="dy", value=dy(grid_id), rc=rc) + + end if + + end if ! .not.is_restart_bundle if(step == 1) then file_bundle = wrt_int_state%wrtFB(nbdl) @@ -3457,11 +3435,13 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) call ESMF_VMGet(vm=vm, mpiCommunicator=wrt_mpi_comm%mpi_val, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (petCount > 1) then - call write_restart_netcdf(wrtTileFB, trim(tileFileName), .true., wrt_mpi_comm, localPet, rc) - else + !Restrict writing cubed sphere restart files to use serial I/O due to slowness + ! on WCOOS2 when large number of tasks in the write group is used + !if (petCount > 1) then + ! call write_restart_netcdf(wrtTileFB, trim(tileFileName), .true., wrt_mpi_comm, localPet, rc) + !else call write_restart_netcdf(wrtTileFB, trim(tileFileName), .false., wrt_mpi_comm, localPet, rc) - endif + !endif endif return @@ -4649,6 +4629,36 @@ end function trim_suffix !----------------------------------------------------------------------- !&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& !----------------------------------------------------------------------- + subroutine print_att_list(fb, rc) + type(ESMF_FieldBundle), intent(in) :: fb + integer, intent(out) :: rc + + integer :: i + integer :: itemCount + integer :: attCount + character(len=ESMF_MAXSTR) :: fbName, attName + type(ESMF_TypeKind_Flag) :: typekind + + rc = 0 + call ESMF_FieldBundleGet(fb, name=fbName, rc=rc) + + write(0,*)'==== ', trim(fbName) + + call ESMF_AttributeGet(fb, convention="NetCDF", purpose="FV3", & + attnestflag=ESMF_ATTNEST_OFF, count=attCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + do i=1, attCount + call ESMF_AttributeGet(fb, convention="NetCDF", purpose="FV3", & + attnestflag=ESMF_ATTNEST_OFF, attributeIndex=i, name=attName, & + typekind=typekind, itemCount=itemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + write(0,*) i , trim(attName), typekind + + end do + + end subroutine print_att_list ! end module module_wrt_grid_comp ! diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index 1a679f18d..5d222497e 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -430,7 +430,6 @@ subroutine post_getattr_fv3(wrt_int_state,grid_id) if (trim(attName) == 'ncnsto') wrt_int_state%ntrac=varival if (trim(attName) == 'ncld') wrt_int_state%ncld=varival if (trim(attName) == 'nsoil') wrt_int_state%nsoil=varival - if (trim(attName) == 'fhzero') wrt_int_state%fhzero=varival if (trim(attName) == 'imp_physics') wrt_int_state%imp_physics=varival endif else if (typekind==ESMF_TYPEKIND_R4) then @@ -439,9 +438,9 @@ subroutine post_getattr_fv3(wrt_int_state,grid_id) name=trim(attName), value=varr4val, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out - if (trim(attName) == 'dtp') then - wrt_int_state%dtp=varr4val - endif + if (trim(attName) == 'dtp') wrt_int_state%dtp=varr4val + if (trim(attName) == 'fhzero') wrt_int_state%fhzero=varr4val +! print *,'in post_fv3, fhzero=',wrt_int_state%fhzero else if(n>1) then if(trim(attName) =="ak") then if(allocated(wrt_int_state%ak)) deallocate(wrt_int_state%ak) @@ -630,7 +629,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) spval = 9.99e20 ! ! nems gfs has zhour defined - tprec = float(wrt_int_state%fhzero) + tprec = wrt_int_state%fhzero tclod = tprec trdlw = tprec trdsw = tprec diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index 07f059023..27cdf955f 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -1363,6 +1363,11 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) integer :: unit real(kind=8) :: mpi_wtime, tbeg1 ! + integer :: FBCount, i + logical :: isPresent + character(len=esmf_maxstr),allocatable :: itemNameList(:) + type(ESMF_StateItem_Flag), allocatable :: itemTypeList(:) + type(ESMF_FieldBundle) :: fcstExportFB !----------------------------------------------------------------------- !*********************************************************************** !----------------------------------------------------------------------- @@ -1405,6 +1410,39 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) endif endif + ! update fhzero + call ESMF_StateGet(exportState, itemCount=FBCount, rc=rc) + + allocate (itemNameList(FBCount)) + allocate (itemTypeList(FBCount)) + call ESMF_StateGet(exportState, & + itemNameList=itemNameList, & + itemTypeList=itemTypeList, & + rc=rc) + do i=1, FBcount + if (itemTypeList(i) == ESMF_STATEITEM_FIELDBUNDLE) then + call ESMF_StateGet(exportState, itemName=itemNameList(i), & + fieldbundle=fcstExportFB, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeGet(fcstExportFB, convention="NetCDF", purpose="FV3", & + name="fhzero", isPresent=isPresent, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + if (isPresent) then + call ESMF_AttributeSet(fcstExportFB, convention="NetCDF", purpose="FV3", name="fhzero", value=GFS_control%fhzero, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + endif + else + !***### anything but a FieldBundle in the state is unexpected here + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg="Only FieldBundles supported in fcstState.", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + + enddo + if (mype == 0) write(*,'(A,I16,A,F16.6)')'PASS: fcstRUN phase 2, n_atmsteps = ', & n_atmsteps,' time is ',mpi_wtime()-tbeg1 !