reentrant: AMReX.cpp

Src/Base/AMReX.cpp

@@ -52,11 +52,20 @@ namespace amrex {
 namespace system
 {
     std::string exename;
-    int verbose = 0;
-    int signal_handling = 1;
+    int verbose;
+    int signal_handling;
 }
 }
 
+namespace {
+    std::new_handler prev_new_handler;
+    typedef void (*SignalHandler)(int);
+    SignalHandler prev_handler_sigsegv;
+    SignalHandler prev_handler_sigint;
+    SignalHandler prev_handler_sigabrt;
+    SignalHandler prev_handler_sigfpe;
+}
+
 std::string amrex::Version ()
 {
 #ifdef AMREX_GIT_VERSION
@@ -273,6 +282,10 @@ void
 amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
                    MPI_Comm mpi_comm, const std::function<void()>& func_parm_parse)
 {
+    system::exename.clear();
+    system::verbose = 0;
+    system::signal_handling = 1;
+
     ParallelDescriptor::StartParallel(&argc, &argv, mpi_comm);
 
 #ifdef AMREX_PMI
@@ -282,7 +295,7 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
     //
     // Make sure to catch new failures.
     //
-    std::set_new_handler(amrex::OutOfMemory);
+    prev_new_handler = std::set_new_handler(amrex::OutOfMemory);
 
     if (argc > 0)
     {
@@ -306,8 +319,10 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
 #endif
 
 #ifdef BL_USE_MPI3
-    BL_MPI_REQUIRE( MPI_Win_create_dynamic(MPI_INFO_NULL, MPI_COMM_WORLD, &ParallelDescriptor::cp_win) );
-    BL_MPI_REQUIRE( MPI_Win_create_dynamic(MPI_INFO_NULL, MPI_COMM_WORLD, &ParallelDescriptor::fb_win) );
+    BL_MPI_REQUIRE( MPI_Win_create_dynamic(MPI_INFO_NULL, ParallelDescriptor::Communicator(),
+                                           &ParallelDescriptor::cp_win) );
+    BL_MPI_REQUIRE( MPI_Win_create_dynamic(MPI_INFO_NULL, ParallelDescriptor::Communicator(),
+                                           &ParallelDescriptor::fb_win) );
 #endif
 
     while ( ! The_Initialize_Function_Stack.empty())
@@ -342,6 +357,8 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
                 ParmParse::Initialize(argc-2,argv+2,argv[1]);
             }
         }
+    } else {
+        ParmParse::Initialize(0,0,0);
     }
 
     if (func_parm_parse) {
@@ -357,9 +374,11 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
         if (system::signal_handling)
         {
             // We could save the singal handlers and restore them in Finalize.
-            signal(SIGSEGV, BLBackTrace::handler); // catch seg falult
-            signal(SIGINT,  BLBackTrace::handler);
-            signal(SIGABRT, BLBackTrace::handler);
+            prev_handler_sigsegv = signal(SIGSEGV, BLBackTrace::handler); // catch seg falult
+            prev_handler_sigint = signal(SIGINT,  BLBackTrace::handler);
+            prev_handler_sigabrt = signal(SIGABRT, BLBackTrace::handler);
+
+            prev_handler_sigfpe = SIG_ERR;
 
             int invalid = 0, divbyzero=0, overflow=0;
             pp.query("fpe_trap_invalid", invalid);
@@ -373,7 +392,7 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
 #if !defined(__PGI) || (__PGIC__ >= 16)
             if (flags != 0) {
                 feenableexcept(flags);  // trap floating point exceptions
-                signal(SIGFPE,  BLBackTrace::handler);
+                prev_handler_sigfpe = signal(SIGFPE,  BLBackTrace::handler);
             }
 #endif
 #endif
@@ -489,18 +508,33 @@ amrex::Finalize (bool finalize_parallel)
     }
 #endif
 
+    amrex_mempool_finalize();
+
 #ifdef BL_MEM_PROFILING
     MemProfiler::report("Final");
+    MemProfiler::Finalize();
 #endif
-    
+
     ParallelDescriptor::EndTeams();
 
     ParallelDescriptor::EndSubCommunicator();
 
+#ifndef BL_AMRPROF
+    if (system::signal_handling)
+    {
+        if (prev_handler_sigsegv != SIG_ERR) signal(SIGSEGV, prev_handler_sigsegv);
+        if (prev_handler_sigint != SIG_ERR) signal(SIGINT, prev_handler_sigint);
+        if (prev_handler_sigabrt != SIG_ERR) signal(SIGABRT, prev_handler_sigabrt);
+        if (prev_handler_sigfpe != SIG_ERR) signal(SIGFPE, prev_handler_sigfpe);
+    }
+#endif
+
 #ifdef BL_USE_UPCXX
     upcxx::finalize();
 #endif
 
+    std::set_new_handler(prev_new_handler);
+
     if (finalize_parallel) {
 #if defined(BL_USE_FORTRAN_MPI)
 	bl_fortran_mpi_comm_free();

Src/Base/AMReX_MemPool.H

@@ -3,6 +3,7 @@
 
 extern "C" {
     void  amrex_mempool_init ();
+    void  amrex_mempool_finalize ();
     void* amrex_mempool_alloc (size_t n);
     void  amrex_mempool_free (void* p);
     void  amrex_mempool_get_stats (int& mp_min, int& mp_max, int& mp_tot);  // min, max & tot in MB

Src/Base/AMReX_MemPool.cpp

@@ -31,13 +31,13 @@ namespace
 #else
     static int init_snan = 0;
 #endif
+    static bool initialized = false;
 }
 
 extern "C" {
 
-void amrex_mempool_init()
+void amrex_mempool_init ()
 {
-    static bool initialized = false;
     if (!initialized)
     {
 	initialized = true;
@@ -78,6 +78,12 @@ void amrex_mempool_init()
     }
 }
 
+void amrex_mempool_finalize ()
+{
+    initialized = false;
+    the_memory_pool.clear();
+}
+
 void* amrex_mempool_alloc (size_t nbytes)
 {
 #ifdef _OPENMP

Src/Base/AMReX_MemProfiler.H

@@ -30,6 +30,8 @@ public:
 
     static void report (const std::string& prefix = std::string());
 
+    static void Finalize ();
+
 private:
 
     MemProfiler (const MemProfiler&) = delete;

Src/Base/AMReX_MemProfiler.cpp

@@ -44,11 +44,23 @@ MemProfiler::add (const std::string& name, std::function<NBuildsInfo()>&& f)
     mprofiler.the_funcs_builds.push_back(std::move(f));
 }
 
+namespace {
+    std::unique_ptr<MemProfiler> the_instance = nullptr;
+}
+
 MemProfiler& 
 MemProfiler::getInstance ()
 {
-    static MemProfiler the_instance;
-    return the_instance;
+    if (the_instance == nullptr) {
+        the_instance.reset(new MemProfiler());
+    }
+    return *the_instance;
+}
+
+void
+MemProfiler::Finalize ()
+{
+    the_instance.reset();
 }
 
 void

Src/Base/AMReX_ParmParse.cpp

@@ -1025,6 +1025,7 @@ ParmParse::Initialize (int         argc,
     {
 	amrex::Error("ParmParse::Initialize(): already initialized!");
     }
+
     ppinit(argc, argv, parfile, g_table);
 
     amrex::ExecOnFinalize(ParmParse::Finalize);