Allow setting ESMF_Finalize endflag in ESMPy and ESMC/I interfaces

src/Infrastructure/Util/include/ESMC_Util.h

@@ -58,6 +58,11 @@ enum ESMC_Decomp_Flag {ESMC_DECOMP_INVALID=0,
                        ESMC_DECOMP_BALANCED, ESMC_DECOMP_RESTFIRST,
                        ESMC_DECOMP_RESTLAST, ESMC_DECOMP_CYCLIC};
 
+// end flag
+typedef enum ESMC_End_Flag { ESMC_END_NORMAL=1,
+                             ESMC_END_KEEPMPI,
+                             ESMC_END_ABORT} ESMC_End_Flag;
+
 enum ESMC_ExtrapMethod_Flag {ESMC_EXTRAPMETHOD_NONE=0,
                              ESMC_EXTRAPMETHOD_NEAREST_STOD,
                              ESMC_EXTRAPMETHOD_NEAREST_IDAVG,

src/Superstructure/ESMFMod/include/ESMCI_Init.h

@@ -28,6 +28,7 @@
 // !USES:
 #include "ESMCI_Macros.h"
 #include "ESMCI_Calendar.h"
+#include "ESMCI_Util.h"
 
 // public globals, filled in by ESMC_Initialize()
 //  and used by MPI_Init().   set once, treat as read-only!
@@ -53,6 +54,8 @@ int ESMCI_Initialize(int argc, char **argv,
 
 int ESMCI_Finalize(void);
 
+int ESMCI_Finalize(ESMC_End_Flag endFlag);
+
 
 // prototypes for fortran interface routines
 extern "C" {
@@ -63,7 +66,7 @@ extern "C" {
                                         ESMC_LogKind_Flag *defaultLogType,
                                         int *rc, ESMCI_FortranStrLenArg count1,
                                         ESMCI_FortranStrLenArg count2);
-   void FTN_X(f_esmf_frameworkfinalize)(int *rc);
+   void FTN_X(f_esmf_frameworkfinalize)(int *rc, ESMC_End_Flag *endFlag);
 };
 
 

src/Superstructure/ESMFMod/include/ESMC_Init.h

@@ -29,6 +29,7 @@
 #define ESMC_Init_H
 
 #include "ESMC_Arg.h"
+#include "ESMC_Util.h"
 
 // identifier list for optional ESMC arguments
 enum {
@@ -125,6 +126,39 @@ extern "C" {
 } // extern "C"
 #endif
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+//-----------------------------------------------------------------------------
+//BOP
+// !IROUTINE: ESMC_FinalizeWithFlag - Finalize the ESMF Framework and specify the type of finalization.
+//
+// !INTERFACE:
+  int ESMC_FinalizeWithFlag(
+    ESMC_End_Flag endFlag); // enumerator for exit action (see below)
+
+// !RETURN VALUE:
+//  Return code; equals ESMF_SUCCESS if there are no errors.
+//
+// !DESCRIPTION:
+// This must be called once on each PET before the application exits to
+// allow ESMF to flush buffers, close open connections, and release
+// internal resources cleanly.
+//
+// The \texttt{endFlag} argument has one of three options:
+// \being{description}
+//   \item [\texttt{ESMC_END_NORMAL}]
+//      Finalize normally.
+//   \item [\texttt{ESMC_END_KEEPMPI}]
+//      Finalize normally without finalizing MPI.
+//   \item [\texttt{ESMC_END_ABORT}]
+//      Abort on finalization.
+// \end{description}
+//EOP
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
 
 #ifdef __cplusplus
 extern "C" {

src/Superstructure/ESMFMod/interface/ESMCI_Init.C

@@ -166,10 +166,35 @@ char **globalargv;
 //EOP
 
     int rc;
+    ESMC_End_Flag endflag=ESMC_END_NORMAL;
 
-    FTN_X(f_esmf_frameworkfinalize)(&rc);
+    FTN_X(f_esmf_frameworkfinalize)(&rc, &endflag);
 
     return rc;
 
- } // end ESMCI_FrameworkFinallize
+ } // end ESMCI_Finalize
 
+//-----------------------------------------------------------------------------
+//BOP
+// !IROUTINE:  ESMCI_Finalize - Finalize the ESMF Framework
+//
+// !INTERFACE:
+      int ESMCI_Finalize(
+//
+// !RETURN VALUE:
+//    int error return code
+//
+// !ARGUMENTS:
+      ESMC_End_Flag endFlag) { // in - optional end action kind
+//
+// !DESCRIPTION:
+//
+//EOP
+
+    int rc;
+
+    FTN_X(f_esmf_frameworkfinalize)(&rc, &endFlag);
+
+    return rc;
+
+ } // end ESMCI_Finalize

src/Superstructure/ESMFMod/interface/ESMC_Init.C

@@ -123,7 +123,33 @@ extern "C" {
   } // end ESMC_Initialize
 //-----------------------------------------------------------------------------
 
-  //-----------------------------------------------------------------------------
+ //-----------------------------------------------------------------------------
+//BOP
+// !IROUTINE:  ESMC_Finalize - Finalize the ESMF Framework
+//
+// !INTERFACE:
+  int ESMC_FinalizeWithFlag(
+//
+// !RETURN VALUE:
+//  int return code
+//
+// !ARGUMENTS:
+    ESMC_End_Flag endFlag){
+//  
+// !DESCRIPTION:
+//
+//EOP
+
+    int localrc;
+
+    localrc = ESMCI_Finalize(endFlag);
+
+    return localrc;
+
+  } // end ESMC_FinalizeWithFlag
+//-----------------------------------------------------------------------------
+
+//-----------------------------------------------------------------------------
 //BOP
 // !IROUTINE:  ESMC_Finalize - Finalize the ESMF Framework
 //

src/Superstructure/ESMFMod/interface/ESMF_Init_C.F90

@@ -59,14 +59,16 @@ subroutine f_esmf_frameworkinitialize(lang, configFileName, &
 
    end subroutine f_esmf_frameworkinitialize
 
-   subroutine f_esmf_frameworkfinalize(rc)
+   subroutine f_esmf_frameworkfinalize(rc, endFlag)
        use ESMF_CompMod
        use ESMF_InitMod
+       use ESMF_UtilTypesMod
 
        implicit none
 
        integer :: rc
+       type(ESMF_End_Flag) :: endFlag
 
-       call ESMF_Finalize(rc=rc)
+       call ESMF_Finalize(endflag=endFlag, rc=rc)
 
    end subroutine f_esmf_frameworkfinalize

src/addon/esmpy/src/esmpy/api/constants.py

@@ -118,6 +118,23 @@ class DecompFlag(IntEnum):
     Decompose elements cyclically across DEs.
     """
 
+class EndAction(IntEnum):
+    """
+    Specify what action to take on finalization.
+    """
+    NORMAL = 1
+    """
+    Indicates ESMF to finalize normally.
+    """
+    KEEP_MPI = 2
+    """
+    Indicates ESMF to finalize normally, but leave MPI initialized.
+    """
+    ABORT = 3
+    """
+    Indicates ESMF to abort on finalization.
+    """
+
 # ExtrapMethod
 class ExtrapMethod(IntEnum):
     """

src/addon/esmpy/src/esmpy/api/esmpymanager.py

@@ -87,11 +87,13 @@ class Manager(object):
 
     :param bool debug: outputs logging information to ESMF logfiles. If
         ``None``, defaults to False.
+    :param bool endFlag: determines the action to take on ESMF finalization.
+        See :class:`~esmpy.api.constants.EndAction` docstring for details. Defaults to ``EndAction.NORMAL``.
     '''
     # The singleton instance for this class
     __singleton = None
 
-    def __new__(cls, debug=False):
+    def __new__(cls, debug=False, endFlag=EndAction.NORMAL):
         '''
         Returns the singleton instance of this class, creating it if it does 
         not already exist.
@@ -106,7 +108,7 @@ def __new__(cls, debug=False):
         return cls.__singleton
 
 
-    def __init__(self, debug=False):
+    def __init__(self, debug=False, endFlag=EndAction.NORMAL):
         # Return no-op
         if self.__esmp_finalized:
             return
@@ -121,6 +123,7 @@ def __init__(self, debug=False):
             ESMP_Initialize(logkind=logkind)
             import atexit; atexit.register(self.__del__)
             self.__esmp_initialized = True
+            self.__esmp_end_flag = endFlag
 
             # set information related to the ESMF Virtual Machine
             vm = ESMP_VMGetGlobal()
@@ -183,7 +186,7 @@ def __del__(self):
             return
 
         # Call ESMP_Finalize and set flags indicating this has been done
-        ESMP_Finalize()
+        ESMP_Finalize(self.__esmp_end_flag)
         self.__esmp_initialized = False
         self.__esmp_finalized = True
 

src/addon/esmpy/src/esmpy/interface/cbindings.py

@@ -218,16 +218,22 @@ def ESMP_Initialize(logkind = constants.LogKind.MULTI):
 _ESMF.ESMC_Finalize.restype = ct.c_int
 _ESMF.ESMC_Finalize.argtypes = []
 
-def ESMP_Finalize():
+def ESMP_Finalize(endFlag = constants.EndAction.NORMAL):
     """
     Preconditions: ESMF has been initialized.
     Postconditions: ESMF has been finalized, all heap memory has been
-                    released, and all MPI states have been cleaned up.
-                    This method can only be called once per execution,
-                    and must be preceded by one and only one call to
-                    ESMP_Initialize(). \n
+                    released, and, if 'endFlag' is set to NORMAL, all MPI states
+                    have been cleaned up. This method can only be called
+                    once per execution, and must be preceded by one and
+                    only one call to ESMP_Initialize(). \n
+    Arguments:\n
+        EndAction (optional) :: endFlag\n
+            Argument Values:\n
+                (default) NORMAL\n
+                KEEP_MPI\n
+                ABORT\n
     """
-    rc = _ESMF.ESMC_Finalize()
+    rc = _ESMF.ESMC_FinalizeWithFlag(endFlag)
     if rc != constants._ESMP_SUCCESS:
         raise ValueError('ESMC_Finalize() failed with rc = '+str(rc)+'.    '+
                         constants._errmsg)