SENSEI

Integrates the SENSEI in situ framework with AMReX. Adds SENSEI data adaptors, and bridge codes for Amr and AmrMesh classes, and is demonstrated through the Advection AmrLevel and AmrCore SingleVortex Tutorial.
wjge · Oct 15, 2018 · b346e12 · b346e12
1 parent 104d3c1
commit b346e12
Show file tree

Hide file tree

Showing 62 changed files with 25,144 additions and 23 deletions.
diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,15 @@ Docs/AMReXUsersGuide/Visualization/*.pdf
 Docs/sphinx/build
 
 .nfs*
+
+# vi
+*.sw[a-z]
+
+# catalyst/libsim
+Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/*.png
+Tutorials/Amr/Advection_AmrCore/Exec/SingleVortex/*.png
+*.vth
+*.vti
+*.pvd
+*.visit
+*.pvsm
diff --git a/GNUmakefile.in b/GNUmakefile.in
@@ -24,6 +24,9 @@ ifeq ($(USE_HYPRE),TRUE)
       Pdirs += Extern/HYPRE
    endif
 endif
+ifeq ($(USE_SENSEI_INSITU),TRUE)
+	Pdirs += Extern/SENSEI
+endif
 Ppack := $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
 include $(Ppack)
 

diff --git a/Src/Amr/AMReX_Amr.H b/Src/Amr/AMReX_Amr.H
@@ -28,6 +28,9 @@ class BoxDomain;
 template <class T>
 class MFGraph;
 class AmrTask;
+#if defined(BL_USE_SENSEI_INSITU)
+class AmrInSituBridge;
+#endif
 
 /**
 * \brief Manage hierarchy of levels for time-dependent AMR computations.
@@ -86,6 +89,7 @@ public:
 
     //! How are we subcycling?
     const std::string& subcyclingMode() const { return subcycling_mode; }
+
     /**  
     * \brief What is "level" in Amr::timeStep?  This is only relevant if we are still in Amr::timeStep;
     *      it is set back to -1 on leaving Amr::timeStep.
@@ -237,6 +241,7 @@ public:
                                           Real* dt_max, 
                                           Real* est_work, 
                                           int*  cycle_max);
+
     //! Write the plot file to be used for visualization.
     virtual void writePlotFile ();
     int stepOfLastPlotFile () const {return last_plotfile;}
@@ -392,6 +397,10 @@ protected:
     void initSubcycle();
     void initPltAndChk();
 
+    int initInSitu();
+    int updateInSitu();
+    static int finalizeInSitu();
+
     //
     // The data ...
     //
@@ -446,6 +455,7 @@ protected:
     Real             loadbalance_max_fac;
 
     bool             bUserStopRequest;
+
     //
     // The static data ...
     //
@@ -459,6 +469,10 @@ protected:
     //! Array of BoxArrays read in to externally define grid hierarchy at each regrid
     static Vector<BoxArray> regrid_ba;
 
+#if defined(BL_USE_SENSEI_INSITU)
+    static AmrInSituBridge *insitu_bridge;
+#endif
+
 public:
 #ifdef USE_PERILLA
     std::vector<RegionGraph*> graphArray;

diff --git a/Src/Amr/AMReX_Amr.cpp b/Src/Amr/AMReX_Amr.cpp
@@ -61,6 +61,10 @@
 #endif
 #endif
 
+#ifdef BL_USE_SENSEI_INSITU
+#include <AMReX_AmrInSituBridge.H>
+#endif
+
 namespace amrex {
 
 //
@@ -72,8 +76,11 @@ std::list<std::string> Amr::derive_plot_vars;
 std::list<std::string> Amr::derive_small_plot_vars;
 bool                   Amr::first_plotfile;
 bool                   Amr::first_smallplotfile;
-Vector<BoxArray>        Amr::initial_ba;
-Vector<BoxArray>        Amr::regrid_ba;
+Vector<BoxArray>       Amr::initial_ba;
+Vector<BoxArray>       Amr::regrid_ba;
+#ifdef BL_USE_SENSEI_INSITU
+AmrInSituBridge*       Amr::insitu_bridge;
+#endif
 
 namespace
 {
@@ -96,6 +103,7 @@ namespace
     int  regrid_on_restart;
     int  use_efficient_regrid;
     int  plotfile_on_restart;
+    int  insitu_on_restart;
     int  checkpoint_on_restart;
     bool checkpoint_files_output;
     int  compute_new_dt_on_regrid;
@@ -105,6 +113,8 @@ namespace
     VisMF::Header::Version checkpoint_headerversion(VisMF::Header::Version_v1);
 //}
 
+
+
 bool
 Amr::UsingPrecreateDirectories()
 {
@@ -128,14 +138,17 @@ Amr::Initialize ()
     regrid_on_restart        = 0;
     use_efficient_regrid     = 0;
     plotfile_on_restart      = 0;
+    insitu_on_restart        = 0;
     checkpoint_on_restart    = 0;
     checkpoint_files_output  = true;
     compute_new_dt_on_regrid = 0;
     precreateDirectories     = true;
     prereadFAHeaders         = true;
     plot_headerversion       = VisMF::Header::Version_v1;
     checkpoint_headerversion = VisMF::Header::Version_v1;
-
+#ifdef BL_USE_SENSEI_INSITU
+    insitu_bridge            = nullptr;
+#endif
     amrex::ExecOnFinalize(Amr::Finalize);
 
     initialized = true;
@@ -149,6 +162,7 @@ Amr::Finalize ()
     Amr::derive_small_plot_vars.clear();
     Amr::regrid_ba.clear();
     Amr::initial_ba.clear();
+    Amr::finalizeInSitu();
 
     initialized = false;
 }
@@ -249,7 +263,10 @@ Amr::InitAmr ()
     record_run_info_terse  = false;
     bUserStopRequest       = false;
     message_int            = 10;
-
+#ifdef BL_USE_SENSEI_INSITU
+    insitu_bridge          = nullptr;
+#endif
+
     for (int i = 0; i < AMREX_SPACEDIM; i++)
         isPeriodic[i] = false;
 
@@ -260,6 +277,7 @@ Amr::InitAmr ()
     pp.query("regrid_on_restart",regrid_on_restart);
     pp.query("use_efficient_regrid",use_efficient_regrid);
     pp.query("plotfile_on_restart",plotfile_on_restart);
+    pp.query("insitu_on_restart",insitu_on_restart);
     pp.query("checkpoint_on_restart",checkpoint_on_restart);
 
     pp.query("compute_new_dt_on_regrid",compute_new_dt_on_regrid);
@@ -352,6 +370,11 @@ Amr::InitAmr ()
     //
     initPltAndChk();
 
+    //
+    // Setup insitu controls
+    //
+    initInSitu();
+
     //
     // Setup subcycling controls.
     //
@@ -486,6 +509,49 @@ Amr::InitAmr ()
     pp.query("loadbalance_max_fac", loadbalance_max_fac);
 }
 
+int
+Amr::initInSitu()
+{
+#if defined(BL_USE_SENSEI_INSITU)
+    insitu_bridge = new AmrInSituBridge;
+    if (insitu_bridge->initialize())
+    {
+        amrex::ErrorStream() << "Amr::initInSitu : Failed to initialize." << std::endl;
+        amrex::Abort();
+    }
+#endif
+    return 0;
+}
+
+int
+Amr::updateInSitu()
+{
+#if defined(BL_USE_SENSEI_INSITU)
+    if (insitu_bridge && insitu_bridge->update(this))
+    {
+        amrex::ErrorStream() << "Amr::updateInSitu : Failed to update." << std::endl;
+        amrex::Abort();
+    }
+#endif
+    return 0;
+}
+
+int
+Amr::finalizeInSitu()
+{
+#if defined(BL_USE_SENSEI_INSITU)
+    if (insitu_bridge)
+    {
+        if (insitu_bridge->finalize())
+            amrex::ErrorStream() << "Amr::finalizeInSitu : Failed to finalize." << std::endl;
+
+        delete insitu_bridge;
+        insitu_bridge = nullptr;
+    }
+#endif
+    return 0;
+}
+
 bool
 Amr::isStatePlotVar (const std::string& name)
 {
@@ -1112,11 +1178,15 @@ Amr::init (Real strt_time,
     {
         initialInit(strt_time,stop_time);
         checkPoint();
+
         if(plot_int > 0 || plot_per > 0) {
             writePlotFile();
-	}
-	if (small_plot_int > 0 || small_plot_per > 0)
-	    writeSmallPlotFile();
+        }
+
+        if (small_plot_int > 0 || small_plot_per > 0)
+	        writeSmallPlotFile();
+
+        updateInSitu();
     }
 
 #ifdef BL_COMM_PROFILING
@@ -1837,6 +1907,8 @@ Amr::RegridOnly (Real time)
     if (checkpoint_on_restart)
        checkPoint();
 
+    if (insitu_on_restart)
+        updateInSitu();
 }
 
 void
@@ -2326,6 +2398,8 @@ Amr::coarseTimeStep (Real stop_time)
         writeSmallPlotFile();
     }
 
+    updateInSitu();
+
     bUserStopRequest = to_stop;
     if (to_stop)
     {

diff --git a/Src/AmrCore/Make.package b/Src/AmrCore/Make.package
@@ -14,6 +14,5 @@ ifeq ($(USE_PARTICLES), TRUE)
   CEXE_headers += AMReX_AmrParGDB.H AMReX_AmrParticles.H
 endif
 
-
 VPATH_LOCATIONS += $(AMREX_HOME)/Src/AmrCore
 INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/AmrCore
diff --git a/Src/CMakeLists.txt b/Src/CMakeLists.txt
@@ -77,6 +77,10 @@ if ( ENABLE_PARTICLES )
    include (Particle/CMakeLists.txt)
 endif ()
 
+if (ENABLE_SENSEI_INSITU)
+   include (Extern/SENSEI/CMakeLists.txt)
+endif ()
+
 #
 # Here we generate AMReX_BuildInfo.cpp
 # If Python < 2.7, script won't work so

diff --git a/Src/Extern/SENSEI/AMReX_AmrDataAdaptor.H b/Src/Extern/SENSEI/AMReX_AmrDataAdaptor.H
@@ -0,0 +1,43 @@
+#ifndef AMReX_AmrDataAdaptor_h
+#define AMReX_AmrDataAdaptor_h
+
+#include "DataAdaptor.h"
+#include <AMReX_Amr.H>
+
+namespace amrex
+{
+
+class AmrDataAdaptor : public sensei::DataAdaptor
+{
+public:
+  static AmrDataAdaptor* New();
+  senseiTypeMacro(AmrDataAdaptor, sensei::DataAdaptor);
+
+  // set the source of the simulation data
+  int SetDataSource(amrex::Amr *amr);
+
+  // SENSEI API
+  int GetNumberOfMeshes(unsigned int &numMeshes) override;
+  int GetMeshName(unsigned int id, std::string &meshName) override;
+  int GetMesh(const std::string &meshName, bool structureOnly, vtkDataObject *&mesh) override;
+  int GetMeshHasGhostNodes(const std::string &meshName, int &nLayers) override;
+  int AddGhostNodesArray(vtkDataObject* mesh, const std::string &meshName) override;
+  int GetMeshHasGhostCells(const std::string &meshName, int &nLayers) override;
+  int AddGhostCellsArray(vtkDataObject* mesh, const std::string &meshName) override;
+  int AddArray(vtkDataObject* mesh, const std::string &meshName, int association, const std::string &arrayName) override;
+  int GetNumberOfArrays(const std::string &meshName, int association, unsigned int &numberOfArrays) override;
+  int GetArrayName(const std::string &meshName, int association, unsigned int index, std::string &arrayName) override;
+  int ReleaseData() override;
+
+protected:
+  AmrDataAdaptor();
+  ~AmrDataAdaptor();
+
+private:
+  struct InternalsType;
+  InternalsType *Internals;
+};
+
+}
+
+#endif