Allow setting ESMF_Finalize endflag in ESMPy and ESMC/I interfaces

src/Superstructure/ESMFMod/include/ESMCI_Init.h

@@ -51,7 +51,7 @@ int ESMCI_Initialize(ESMC_CalKind_Flag defaultCalendar=ESMC_CALKIND_NOCALENDAR);
 int ESMCI_Initialize(int argc, char **argv,
   ESMC_CalKind_Flag defaultCalendar=ESMC_CALKIND_NOCALENDAR);
 
-int ESMCI_Finalize(void);
+int ESMCI_Finalize(ESMC_Logical keepMpi=ESMF_FALSE);
 
 
 // prototypes for fortran interface routines
@@ -63,7 +63,7 @@ extern "C" {
                                         ESMC_LogKind_Flag *defaultLogType,
                                         int *rc, ESMCI_FortranStrLenArg count1,
                                         ESMCI_FortranStrLenArg count2);
-   void FTN_X(f_esmf_frameworkfinalize)(int *rc);
+   void FTN_X(f_esmf_frameworkfinalize)(int *rc, bool keepMpi);
 };
 
 

src/Superstructure/ESMFMod/include/ESMC_Init.h

@@ -29,6 +29,7 @@
 #define ESMC_Init_H
 
 #include "ESMC_Arg.h"
+#include "ESMC_Util.h"
 
 // identifier list for optional ESMC arguments
 enum {
@@ -125,6 +126,29 @@ extern "C" {
 } // extern "C"
 #endif
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+//-----------------------------------------------------------------------------
+//BOP
+// !IROUTINE: ESMC_FinalizeMPI - Finalize the ESMF Framework and specify whether
+//                               MPI should be finalized
+//
+// !INTERFACE:
+  int ESMC_FinalizeMPI(ESMC_Logical keepMpi);
+
+// !RETURN VALUE:
+//  Return code; equals ESMF_SUCCESS if there are no errors.
+//
+// !DESCRIPTION:
+// This must be called once on each PET before the application exits to
+// allow ESMF to flush buffers, close open connections, and release
+// internal resources cleanly.
+//EOP
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
 
 #ifdef __cplusplus
 extern "C" {

src/Superstructure/ESMFMod/interface/ESMCI_Init.C

@@ -159,15 +159,15 @@ char **globalargv;
 //    int error return code
 //
 // !ARGUMENTS:
-      void) {
+      ESMC_Logical keepMpi) {
 //
 // !DESCRIPTION:
 //
 //EOP
 
     int rc;
 
-    FTN_X(f_esmf_frameworkfinalize)(&rc);
+    FTN_X(f_esmf_frameworkfinalize)(&rc, keepMpi == ESMF_TRUE);
 
     return rc;
 

src/Superstructure/ESMFMod/interface/ESMC_Init.C

@@ -123,7 +123,33 @@ extern "C" {
   } // end ESMC_Initialize
 //-----------------------------------------------------------------------------
 
-  //-----------------------------------------------------------------------------
+ //-----------------------------------------------------------------------------
+//BOP
+// !IROUTINE:  ESMC_Finalize - Finalize the ESMF Framework
+//
+// !INTERFACE:
+  int ESMC_FinalizeMPI(
+//
+// !RETURN VALUE:
+//  int return code
+//
+// !ARGUMENTS:
+    ESMC_Logical keepMpi){
+//  
+// !DESCRIPTION:
+//
+//EOP
+
+    int localrc;
+
+    localrc = ESMCI_Finalize(keepMpi);
+
+    return localrc;
+
+  } // end ESMC_FinalizeMPI
+//-----------------------------------------------------------------------------
+
+//-----------------------------------------------------------------------------
 //BOP
 // !IROUTINE:  ESMC_Finalize - Finalize the ESMF Framework
 //
@@ -143,7 +169,7 @@ extern "C" {
     int localrc;
 
     // todo: it may be better to go directly into F90 instead of using C++
-    localrc = ESMCI_Finalize();
+    localrc = ESMC_FinalizeMPI(ESMF_FALSE);
 
     // todo: use LogErr to do error handling for localrc
 

src/Superstructure/ESMFMod/interface/ESMF_Init_C.F90

@@ -59,14 +59,26 @@ subroutine f_esmf_frameworkinitialize(lang, configFileName, &
 
    end subroutine f_esmf_frameworkinitialize
 
-   subroutine f_esmf_frameworkfinalize(rc)
+   subroutine f_esmf_frameworkfinalize(rc, keepMpi)
        use ESMF_CompMod
        use ESMF_InitMod
+       use ESMF_UtilTypesMod
+       use, intrinsic :: iso_c_binding
 
        implicit none
 
        integer :: rc
+       logical(kind=c_bool), intent(in), optional :: keepMpi
+       type(ESMF_End_Flag) :: endflag
 
-       call ESMF_Finalize(rc=rc)
+       if (present(keepMpi)) then
+         if (keepMpi) then
+            endflag = ESMF_END_KEEPMPI
+         else
+            endflag = ESMF_END_NORMAL
+         endif
+       endif
+
+       call ESMF_Finalize(rc=rc, endflag=endflag)
 
    end subroutine f_esmf_frameworkfinalize

src/addon/esmpy/src/esmpy/api/esmpymanager.py

@@ -87,11 +87,14 @@ class Manager(object):
 
     :param bool debug: outputs logging information to ESMF logfiles. If
         ``None``, defaults to False.
+    :param bool keepMpi: determines whether MPI should be kept active on
+        ESMF finalization. If False, then MPI_Finalize will be called on finalization.
+        If ``None``, defaults to False.
     '''
     # The singleton instance for this class
     __singleton = None
 
-    def __new__(cls, debug=False):
+    def __new__(cls, debug=False, keepMpi=False):
         '''
         Returns the singleton instance of this class, creating it if it does 
         not already exist.
@@ -106,7 +109,7 @@ def __new__(cls, debug=False):
         return cls.__singleton
 
 
-    def __init__(self, debug=False):
+    def __init__(self, debug=False, keepMpi=False):
         # Return no-op
         if self.__esmp_finalized:
             return
@@ -121,6 +124,9 @@ def __init__(self, debug=False):
             ESMP_Initialize(logkind=logkind)
             import atexit; atexit.register(self.__del__)
             self.__esmp_initialized = True
+            self.__esmp_keep_mpi = keepMpi
+            if self.__esmp_keep_mpi is None:
+                self.__esmp_keep_mpi = True
 
             # set information related to the ESMF Virtual Machine
             vm = ESMP_VMGetGlobal()
@@ -183,7 +189,7 @@ def __del__(self):
             return
 
         # Call ESMP_Finalize and set flags indicating this has been done
-        ESMP_Finalize()
+        ESMP_Finalize(self.__esmp_keep_mpi)
         self.__esmp_initialized = False
         self.__esmp_finalized = True
 

src/addon/esmpy/src/esmpy/interface/cbindings.py

@@ -218,16 +218,18 @@ def ESMP_Initialize(logkind = constants.LogKind.MULTI):
 _ESMF.ESMC_Finalize.restype = ct.c_int
 _ESMF.ESMC_Finalize.argtypes = []
 
-def ESMP_Finalize():
+def ESMP_Finalize(keepMpi=False):
     """
     Preconditions: ESMF has been initialized.
     Postconditions: ESMF has been finalized, all heap memory has been
-                    released, and all MPI states have been cleaned up.
-                    This method can only be called once per execution,
-                    and must be preceded by one and only one call to
-                    ESMP_Initialize(). \n
+                    released, and, if 'keepMpi' is False, all MPI states
+                    have been cleaned up. This method can only be called
+                    once per execution, and must be preceded by one and
+                    only one call to ESMP_Initialize(). \n
+    Arguments:\n
+        bool :: keepMpi\n
     """
-    rc = _ESMF.ESMC_Finalize()
+    rc = _ESMF.ESMC_FinalizeMPI(keepMpi)
     if rc != constants._ESMP_SUCCESS:
         raise ValueError('ESMC_Finalize() failed with rc = '+str(rc)+'.    '+
                         constants._errmsg)