ESMF managed threading

.gitmodules

@@ -1,7 +1,9 @@
 [submodule "FV3"]
   path = FV3
-  url = https://github.com/NOAA-EMC/fv3atm
-  branch = develop
+  #url = https://github.com/NOAA-EMC/fv3atm
+  #branch = develop
+  url = https://github.com/junwang-noaa/fv3atm
+  branch = esmf_threading
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3

driver/EARTH_GRID_COMP.F90

@@ -54,18 +54,21 @@ MODULE module_EARTH_GRID_COMP
       use FRONT_HYCOM,      only: HYCOM_SS  => SetServices
 #endif
 #ifdef FRONT_MOM6
-      use FRONT_MOM6,       only: MOM6_SS   => SetServices
+      use FRONT_MOM6,       only: MOM6_SS   => SetServices, &
+                                  MOM6_SV   => SetVM
 #endif
 #ifdef FRONT_CDEPS_DOCN
       use FRONT_CDEPS_DOCN, only: DOCN_SS  => SetServices
 #endif
   ! - Handle build time ICE options:
 #ifdef FRONT_CICE6
-      use FRONT_CICE6,      only: CICE6_SS => SetServices
+      use FRONT_CICE6,      only: CICE6_SS => SetServices, &
+                                  CICE6_SV => SetVM
 #endif
   ! - Handle build time WAV options:
 #ifdef FRONT_WW3
-      use FRONT_WW3,        only: WW3_SS  => SetServices
+      use FRONT_WW3,        only: WW3_SS  => SetServices, &
+                                  WW3_SV  => SetVM
 #endif
   ! - Handle build time LND options:
 #ifdef FRONT_NOAH
@@ -84,7 +87,8 @@ MODULE module_EARTH_GRID_COMP
 #endif
   ! - Mediator
 #ifdef FRONT_CMEPS
-      use MED,              only: MED_SS     => SetServices
+      use MED,              only: MED_SS     => SetServices, &
+                                  MED_SV     => SetVM
 #endif
 !
 !-----------------------------------------------------------------------
@@ -226,12 +230,14 @@ subroutine SetModelServices(driver, rc)
         character(ESMF_MAXSTR)          :: name
         type(ESMF_GridComp)             :: comp
         type(ESMF_Config)               :: config
+        type(ESMF_Info)                 :: info
         character(len=32), allocatable  :: compLabels(:)
         integer, allocatable            :: petList(:)
         character(len=10)               :: value
         character(len=20)               :: model, prefix
         character(len=160)              :: msg
         integer                         :: petListBounds(2)
+        integer                         :: ompNumThreads
         integer                         :: componentCount
         type(NUOPC_FreeFormat)          :: attrFF, fdFF
         logical                         :: found_comp
@@ -332,17 +338,42 @@ subroutine SetModelServices(driver, rc)
             petList(j-petListBounds(1)+1) = j ! PETs are 0 based
           enddo
 
+! *** read in number of OpenMP threads for this component
+          call ESMF_ConfigGetAttribute(config, ompNumThreads, &
+            label=trim(prefix)//"_omp_num_threads:", default=-1, rc=rc)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
+
+! *** create info object
+          info = ESMF_InfoCreate(rc=rc)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
+
+! *** set up PePerPet NUOPC hints
+          if (ompNumThreads /= -1) then
+            call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", &
+              value=ompNumThreads, rc=rc)
+            if (ChkErr(rc,__LINE__,u_FILE_u)) return
+          endif
+
           found_comp = .false.
 #ifdef FRONT_FV3
           if (trim(model) == "fv3") then
             call NUOPC_DriverAddComp(driver, trim(prefix), FV3_SS, &
-              petList=petList, comp=comp, rc=rc)
+              info=info, petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
             found_comp = .true.
           end if
 #endif
 #if defined FRONT_CDEPS_DATM
           if (trim(model) == "datm" ) then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
             call NUOPC_DriverAddComp(driver, trim(prefix), DATM_SS, &
               petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -351,6 +382,15 @@ subroutine SetModelServices(driver, rc)
 #endif
 #ifdef FRONT_HYCOM
           if (trim(model) == "hycom") then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
             call NUOPC_DriverAddComp(driver, trim(prefix), HYCOM_SS, &
               petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -360,13 +400,22 @@ subroutine SetModelServices(driver, rc)
 #ifdef FRONT_MOM6
           if (trim(model) == "mom6") then
             call NUOPC_DriverAddComp(driver, trim(prefix), MOM6_SS, &
-              petList=petList, comp=comp, rc=rc)
+               MOM6_SV, info=info, petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
             found_comp = .true.
           end if
 #endif
 #ifdef FRONT_CDEPS_DOCN
           if (trim(model) == "docn") then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
             call NUOPC_DriverAddComp(driver, trim(prefix), DOCN_SS, &
               petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -376,21 +425,30 @@ subroutine SetModelServices(driver, rc)
 #ifdef FRONT_CICE6
           if (trim(model) == "cice6") then
             call NUOPC_DriverAddComp(driver, trim(prefix), CICE6_SS, &
-              petList=petList, comp=comp, rc=rc)
+              CICE6_SV, info=info, petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
             found_comp = .true.
           end if
 #endif
 #ifdef FRONT_WW3
           if (trim(model) == "ww3") then
             call NUOPC_DriverAddComp(driver, trim(prefix), WW3_SS, &
-              petList=petList, comp=comp, rc=rc)
+              WW3_SV, info=info, petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
             found_comp = .true.
           end if
 #endif
 #ifdef FRONT_NOAH
           if (trim(model) == "noah") then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
             call NUOPC_DriverAddComp(driver, trim(prefix), NOAH_SS, &
               petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -399,6 +457,15 @@ subroutine SetModelServices(driver, rc)
 #endif
 #ifdef FRONT_LIS
           if (trim(model) == "lis") then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
             call NUOPC_DriverAddComp(driver, trim(prefix), LIS_SS, &
               petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -407,6 +474,15 @@ subroutine SetModelServices(driver, rc)
 #endif
 #ifdef FRONT_IPE
           if (trim(model) == "ipe") then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
             call NUOPC_DriverAddComp(driver, trim(prefix), IPE_SS, &
               petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -415,6 +491,15 @@ subroutine SetModelServices(driver, rc)
 #endif
 #ifdef FRONT_GOCART
           if (trim(model) == "gocart") then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
             call NUOPC_DriverAddComp(driver, trim(prefix), GOCART_SS, &
               petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -424,7 +509,7 @@ subroutine SetModelServices(driver, rc)
 #ifdef FRONT_CMEPS
           if (trim(model) == "cmeps") then
             call NUOPC_DriverAddComp(driver, trim(prefix), MED_SS, &
-              petList=petList, comp=comp, rc=rc)
+               MED_SV, info=info, petList=petList, comp=comp, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
             found_comp = .true.
           end if
@@ -441,6 +526,8 @@ subroutine SetModelServices(driver, rc)
 
           ! clean-up
           deallocate(petList)
+          call ESMF_InfoDestroy(info, rc=rc)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
         enddo
 
         ! clean-up

driver/UFS.F90

@@ -76,8 +76,6 @@ PROGRAM UFS
       TYPE(ESMF_Clock) :: CLOCK_MAIN                                       !<-- The ESMF time management clock
 !
       TYPE(ESMF_Config) :: CF_MAIN                                         !<-- The Configure object
-!
-      LOGICAL :: PRINT_ESMF                                                !<-- Flag for ESMF PET files
 !
       CHARACTER(ESMF_MAXSTR) :: MESSAGE_CHECK
 !
@@ -89,28 +87,20 @@ PROGRAM UFS
 !-----------------------------------------------------------------------
 !
 !-----------------------------------------------------------------------
-!***  Check if we want ESMF PET files or not
-!-----------------------------------------------------------------------
-!
-      CALL CHECK_ESMF_PET(PRINT_ESMF)
-!
-!-----------------------------------------------------------------------
 !***  Initialize the ESMF framework.
 !-----------------------------------------------------------------------
 !
-      IF(PRINT_ESMF) THEN
-        CALL ESMF_Initialize(VM             =VM                         & !<-- The ESMF Virtual Machine
-                            ,defaultCalKind =ESMF_CALKIND_GREGORIAN     & !<-- Set up the default calendar.
-                            ,logkindflag    =ESMF_LOGKIND_MULTI         & !<-- Define multiple log error output files;
-                            ,rc             =RC)
-        ESMF_ERR_ABORT(RC)
-      ELSE
-        CALL ESMF_Initialize(VM             =VM                         & !<-- The ESMF Virtual Machine
-                            ,defaultCalKind =ESMF_CALKIND_GREGORIAN     & !<-- Set up the default calendar.
-                            ,logkindflag    =ESMF_LOGKIND_NONE          & !<-- Define no log error output files;
-                            ,rc             =RC)
-        ESMF_ERR_ABORT(RC)
-      ENDIF
+      CALL ESMF_Initialize(configFileName="nems.configure"              & !<-- top level configuration
+                          ,defaultCalKind =ESMF_CALKIND_GREGORIAN       & !<-- Set up the default calendar.
+                          ,VM             =VM                           & !<-- The ESMF Virtual Machine
+                          ,rc             =RC)
+      ESMF_ERR_ABORT(RC)
+
+#if 0
+!***  Useful when debugging ESMF-managed threading and resource control
+      call ESMF_VMLog(vm, rc=rc)
+      ESMF_ERR_ABORT(RC)
+#endif
 !
 !-----------------------------------------------------------------------
 !***  Extract the MPI task ID.
@@ -142,12 +132,10 @@ PROGRAM UFS
 !     CALL ESMF_LogWrite(MESSAGE_CHECK,ESMF_LOGMSG_INFO,rc=RC)
 ! ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
 !
-      IF(PRINT_ESMF) THEN
-        CALL ESMF_LogSet(flush      =.false.                            &
-                        ,trace      =.false.                            &
-                        ,rc         =RC)
-        ESMF_ERR_ABORT(RC)
-      ENDIF
+      CALL ESMF_LogSet(flush      =.false.                            &
+                      ,trace      =.false.                            &
+                      ,rc         =RC)
+      ESMF_ERR_ABORT(RC)
 
 ! ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
 !
@@ -479,39 +467,6 @@ PROGRAM UFS
 !
 !-----------------------------------------------------------------------
 !
-contains
-      subroutine check_esmf_pet(print_esmf)
-
-      implicit none
-      integer :: i,n
-      character *256 :: c1,c2
-      logical :: opened,print_esmf
-
-      do n=101,201
-        inquire(n,opened=opened)
-        if(.not.opened)then
-          open(n,file='model_configure',status='old')  !<-- Open configure file
-          exit
-        endif
-      enddo
-
-      print_esmf=.false.
-
-      do i=1,10000
-        read(n,*,end=22)c1,c2
-        if(c1(1:10) == 'print_esmf') then              !<-- Search for print_esmf flag
-          if( c2 == 'true'   .or.          &           !<-- Check if print_esmf is true or false
-              c2 == '.true.' .or.          &
-              c2 == 'TRUE'   .or.          &
-              c2 == '.TRUE.' ) print_esmf=.true.
-          exit
-        endif
-      enddo
-  22  close(n)
-      return
-
-      end subroutine check_esmf_pet
-
       END PROGRAM UFS
 !
 !-----------------------------------------------------------------------

modulefiles/ufs_hera.intel

@@ -20,6 +20,7 @@ module load hpc/1.1.0
 
 module load hpc-intel/18.0.5.274
 module load hpc-impi/2018.0.4
+#module swap impi/2021.3.0
 
 module load ufs_common
 
@@ -32,4 +33,4 @@ setenv FC mpiifort
 setenv CMAKE_Platform hera.intel
 
 # use shared memory and OpenFabrics Alliance (OFA) fabric with Intel MPI to circumvent RDMA-related bug in DAPL.
-setenv I_MPI_FABRICS shm:ofa
+#setenv I_MPI_FABRICS shm:ofa

modulefiles/ufs_hera.intel_debug

@@ -20,6 +20,7 @@ module load hpc/1.1.0
 
 module load hpc-intel/18.0.5.274
 module load hpc-impi/2018.0.4
+#module swap impi/2021.3.0
 
 module load ufs_common_debug
 

tests/parm/datm_cdeps_configure.IN

@@ -1,4 +1,3 @@
-print_esmf:              .true.
 start_year:              @[SYEAR]
 start_month:             @[SMONTH]
 start_day:               @[SDAY]

tests/parm/model_configure.IN

@@ -1,4 +1,3 @@
-print_esmf:              .true.
 start_year:              @[SYEAR]
 start_month:             @[SMONTH]
 start_day:               @[SDAY]

tests/parm/model_configure_fhout.IN

@@ -1,5 +1,3 @@
-total_member:            @[ENS_NUM]
-print_esmf:              .true.
 PE_MEMBER01:             @[TASKS]
 start_year:              @[SYEAR]
 start_month:             @[SMONTH]

tests/parm/model_configure_hafs.IN

@@ -1,4 +1,3 @@
-print_esmf:              .true.
 start_year:              2018
 start_month:             10
 start_day:               15