Add Caliper support

src/MC_Fast_Timer.cc

@@ -5,6 +5,10 @@
 #include "Globals.hh"
 #include "portability.hh"
 
+#ifdef USE_CALIPER
+#include <adiak.hpp>
+#endif
+
 const char *mc_fast_timer_names[MC_Fast_Timer::Num_Timers] =
 {
     "main",
@@ -101,6 +105,12 @@ void MC_Fast_Timer_Container::Cumulative_Report(int mpi_rank, int num_ranks, MPI
                         "Figure Of Merit",
                         (numSegments / (max_clock[cycleTracking_Index]*1e-6)),
                         "[Num Segments / Cycle Tracking Time]" );
+
+#ifdef USE_CALIPER
+        adiak::value("numSegments", numSegments);
+        adiak::value("CycleTrackingTime", max_clock[cycleTracking_Index]*1e-6);
+        adiak::value("FigureOfMerit", numSegments / (max_clock[cycleTracking_Index]*1e-6));
+#endif
     }
 }
 

src/MC_Fast_Timer.hh

@@ -6,6 +6,10 @@
 #include <chrono>
 #endif
 
+#ifdef USE_CALIPER
+#include <caliper/cali.h>
+#endif
+
 #include "portability.hh"   // needed for uint64_t in this file
 #include "utilsMpi.hh"      // needed for MPI_Comm type in this file
 
@@ -44,12 +48,13 @@ class MC_Fast_Timer
 class MC_Fast_Timer_Container
 {
 public:
-    MC_Fast_Timer_Container() {} ; // constructor
+    MC_Fast_Timer_Container()
+        {} ; // constructor
     void Cumulative_Report(int mpi_rank, int num_ranks, MPI_Comm comm_world, uint64_t numSegments);
     void Last_Cycle_Report(int report_time, int mpi_rank, int num_ranks, MPI_Comm comm_world);
     void Clear_Last_Cycle_Timers();
     MC_Fast_Timer  timers[MC_Fast_Timer::Num_Timers];  // timers for various routines
-
+    
 private:
     void Print_Cumulative_Heading(int mpi_rank);
     void Print_Last_Cycle_Heading(int mpi_rank);
@@ -87,27 +92,49 @@ extern const char *mc_fast_timer_names[MC_Fast_Timer::Num_Timers];
       #define MC_FASTTIMER_GET_LASTCYCLE(timerIndex) (float)mcco->fast_timer->timers[timerIndex].lastCycleClock / 1000000.
 
    #else // else CHRONO_MISSING is not defined, so high resolution clock is available
-
-      #define MC_FASTTIMER_START(timerIndex) \
-          if (omp_get_thread_num() == 0) { \
-              mcco->fast_timer->timers[timerIndex].startClock = std::chrono::high_resolution_clock::now(); \
-          }
-
-      #define MC_FASTTIMER_STOP(timerIndex) \
-          if ( omp_get_thread_num() == 0 ) { \
-              mcco->fast_timer->timers[timerIndex].stopClock = std::chrono::high_resolution_clock::now(); \
-              mcco->fast_timer->timers[timerIndex].lastCycleClock += \
-                std::chrono::duration_cast<std::chrono::microseconds> \
-		(mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
-              mcco->fast_timer->timers[timerIndex].cumulativeClock += \
-	        std::chrono::duration_cast<std::chrono::microseconds> \
-	        (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
-              mcco->fast_timer->timers[timerIndex].numCalls++;		\
-          }
+      #ifdef USE_CALIPER
+         #define MC_FASTTIMER_START(timerIndex) \
+             if (omp_get_thread_num() == 0) { \
+                 cali_begin_region(mc_fast_timer_names[timerIndex]); \
+                 mcco->fast_timer->timers[timerIndex].startClock = std::chrono::high_resolution_clock::now(); \
+             }
+
+         #define MC_FASTTIMER_STOP(timerIndex) \
+             if ( omp_get_thread_num() == 0 ) { \
+                 mcco->fast_timer->timers[timerIndex].stopClock = std::chrono::high_resolution_clock::now(); \
+                 mcco->fast_timer->timers[timerIndex].lastCycleClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].cumulativeClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].numCalls++;		\
+                 cali_end_region(mc_fast_timer_names[timerIndex]); \
+             }
+
+      #else // not defined USE_CALIPER
+
+         #define MC_FASTTIMER_START(timerIndex) \
+             if (omp_get_thread_num() == 0) { \
+                 mcco->fast_timer->timers[timerIndex].startClock = std::chrono::high_resolution_clock::now(); \
+             }
+
+         #define MC_FASTTIMER_STOP(timerIndex) \
+             if ( omp_get_thread_num() == 0 ) { \
+                 mcco->fast_timer->timers[timerIndex].stopClock = std::chrono::high_resolution_clock::now(); \
+                 mcco->fast_timer->timers[timerIndex].lastCycleClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].cumulativeClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].numCalls++;		\
+             }
+
+      #endif // end ifdef USE_CALIPER else branch
 
       #define MC_FASTTIMER_GET_LASTCYCLE(timerIndex) (float)mcco->fast_timer->timers[timerIndex].lastCycleClock / 1000000.
 
-
    #endif // end ifdef CHRONO_MISSING else section
 #endif // end if DISABLE_TIMERS
 

src/Makefile

@@ -84,6 +84,14 @@
 #                   with some Clang compilers, some older Gnu compilers on BG/Q
 #                   and older Intel compilers.
 #         
+# -DUSE_CALIPER     Define this to enable Caliper instrumentation. Caliper
+#                   is a performance profiling / analysis library for
+#                   tracing, sampling, HW counter measurements, and much more.
+#                       When enabled, Quicksilver will link in the Caliper
+#                   library and Adiak libraries and export its timed regions
+#                   to Caliper and store run metadata in Adiak. See
+#                   https://github.com/LLNL/Caliper for more information.
+#         
 # -DUSE_NVTX        Define this for some extra NVProf profiling information.
 #                   It will create regions that can be visualized in NVVP.
 #
@@ -139,7 +147,6 @@ LDFLAGS = -fgpu-rdc --hip-link --offload-arch=gfx90a
 #CPPFLAGS = $(OPENMP_FLAGS)
 #LDFLAGS = $(OPENMP_LDFLAGS) 
 
-
 ###############################################################################
 ### GCC -- with MPI and OpenMP 
 ###############################################################################
@@ -154,6 +161,26 @@ LDFLAGS = -fgpu-rdc --hip-link --offload-arch=gfx90a
 #LDFLAGS = $(OPENMP_LDFLAGS) 
 
 
+###############################################################################
+### GCC -- with MPI and OpenMP and Caliper support
+###############################################################################
+# ADIAK_DIR = $(spack location --install-dir adiak)
+# CALIPER_DIR = $(spack location --install-dir caliper)
+
+# CALIPER_FLAGS = -I${CALIPER_DIR}/include -I${ADIAK_DIR}/include -DUSE_CALIPER
+# CALIPER_LDFLAGS = -Wl,-rpath ${CALIPER_DIR}/lib64 -Wl,-rpath ${ADIAK_DIR}/lib -L${CALIPER_DIR}/lib64 -L${ADIAK_DIR}/lib -lcaliper -ladiak
+
+# OPENMP_FLAGS = -DHAVE_OPENMP -fopenmp
+# OPENMP_LDFLAGS = -fopenmp
+# MPI_FLAGS = -DHAVE_MPI
+# OPTFLAGS = -g -O2
+
+# CXX=mpicxx
+# CXXFLAGS = -std=c++11 $(OPTFLAGS) -Wpedantic
+# CPPFLAGS = $(MPI_FLAGS) $(OPENMP_FLAGS) $(CALIPER_FLAGS)
+# LDFLAGS = $(OPENMP_LDFLAGS) $(CALIPER_LDFLAGS)
+
+
 ###############################################################################
 # LLNL LC BG/Q Comilers                                                       #
 ###############################################################################

src/Parameters.cc

@@ -33,6 +33,10 @@
 #include "InputBlock.hh"
 #include "utilsMpi.hh"
 
+#ifdef USE_CALIPER
+#include <adiak.hpp>
+#endif
+
 using std::string;
 using std::ifstream;
 using std::make_pair;
@@ -94,6 +98,100 @@ Parameters getParameters(int argc, char** argv)
    return params;
 }
 
+void saveParametersInAdiak(const Parameters& pp)
+{
+#ifdef USE_CALIPER
+   adiak::value("dt", pp.simulationParams.dt);
+   adiak::value("fMax", pp.simulationParams.fMax);
+   adiak::value("inputFile", adiak::path(pp.simulationParams.inputFile));
+   adiak::value("energySpectrum", pp.simulationParams.energySpectrum);
+   adiak::value("boundaryCondition", pp.simulationParams.boundaryCondition);
+   adiak::value("loadBalance", pp.simulationParams.loadBalance);
+   adiak::value("cycleTimers", pp.simulationParams.cycleTimers);
+   adiak::value("debugThreads", pp.simulationParams.debugThreads);
+   adiak::value("lx", pp.simulationParams.lx);
+   adiak::value("ly", pp.simulationParams.ly);
+   adiak::value("lz", pp.simulationParams.lz);
+   adiak::value("nParticles", pp.simulationParams.nParticles);
+   adiak::value("batchSize", pp.simulationParams.batchSize);
+   adiak::value("nBatches", pp.simulationParams.nBatches);
+   adiak::value("nSteps", pp.simulationParams.nSteps);
+   adiak::value("nx", pp.simulationParams.nx);
+   adiak::value("ny", pp.simulationParams.ny);
+   adiak::value("nz", pp.simulationParams.nz);
+   adiak::value("seed", pp.simulationParams.seed);
+   adiak::value("xDom", pp.simulationParams.xDom);
+   adiak::value("yDom", pp.simulationParams.yDom);
+   adiak::value("zDom", pp.simulationParams.zDom);
+   adiak::value("eMax", pp.simulationParams.eMax);
+   adiak::value("eMin", pp.simulationParams.eMin);
+   adiak::value("nGroups", pp.simulationParams.nGroups);
+   adiak::value("lowWeightCutoff", pp.simulationParams.lowWeightCutoff);
+   adiak::value("bTally", pp.simulationParams.balanceTallyReplications);
+   adiak::value("fTally", pp.simulationParams.fluxTallyReplications);
+   adiak::value("cTally", pp.simulationParams.cellTallyReplications);
+   adiak::value("coralBenchmark", pp.simulationParams.coralBenchmark);
+   adiak::value("crossSectionsOut", pp.simulationParams.crossSectionsOut);
+
+   for (size_t i = 0; i < pp.geometryParams.size(); ++i) {
+      std::string prefix("geometry.");
+      prefix.append(std::to_string(i));
+      prefix.append(".");
+      const GeometryParameters& gp = pp.geometryParams[i];
+      adiak::value(prefix+"material", gp.materialName);
+      switch (gp.shape)
+      {
+        case GeometryParameters::BRICK:
+         adiak::value(prefix+"shape", "brick");
+         adiak::value(prefix+"xMax", gp.xMax);
+         adiak::value(prefix+"xMin", gp.xMin);
+         adiak::value(prefix+"yMax", gp.yMax);
+         adiak::value(prefix+"yMin", gp.yMin);
+         adiak::value(prefix+"zMax", gp.zMax);
+         adiak::value(prefix+"zMin", gp.zMin);
+         break;
+        case GeometryParameters::SPHERE:
+         adiak::value(prefix+"shape", "sphere");
+         adiak::value(prefix+"xCenter", gp.xCenter);
+         adiak::value(prefix+"yCenter", gp.yCenter);
+         adiak::value(prefix+"zCenter", gp.zCenter);
+         break;
+        default:
+         qs_assert(false);
+      }   
+   }
+
+   for (const auto& material : pp.materialParams) {
+      std::string prefix("material.");
+      prefix.append(material.first);
+      prefix.append(".");
+      adiak::value(prefix+"mass", material.second.mass);
+      adiak::value(prefix+"nIsotopes", material.second.nIsotopes);
+      adiak::value(prefix+"nReactions", material.second.nReactions);
+      adiak::value(prefix+"sourceRate", material.second.sourceRate);
+      adiak::value(prefix+"totalCrossSection", material.second.totalCrossSection);
+      adiak::value(prefix+"absorptionCrossSection", material.second.absorptionCrossSection);
+      adiak::value(prefix+"fissionCrossSection", material.second.fissionCrossSection);
+      adiak::value(prefix+"scatteringCrossSection", material.second.scatteringCrossSection);
+      adiak::value(prefix+"absorptionCrossSectionRatio", material.second.absorptionCrossSectionRatio);
+      adiak::value(prefix+"fissionCrossSectionRatio", material.second.fissionCrossSectionRatio);
+      adiak::value(prefix+"scatteringCrossSectionRatio", material.second.scatteringCrossSectionRatio);
+   }
+
+   for (const auto& csp : pp.crossSectionParams) {
+      std::string prefix("crossection.");
+      prefix.append(csp.first);
+      prefix.append(".");
+      adiak::value(prefix+"A", csp.second.aa);
+      adiak::value(prefix+"B", csp.second.bb);
+      adiak::value(prefix+"C", csp.second.cc);
+      adiak::value(prefix+"D", csp.second.dd);
+      adiak::value(prefix+"E", csp.second.ee);
+      adiak::value(prefix+"nuBar", csp.second.nuBar);   
+   }
+#endif
+}
+
 void printParameters(const Parameters& pp, ostream& out)
 {
    int rank = -1;
@@ -226,6 +324,8 @@ namespace
       esName[0] = '\0';
       char xsec[1024];
       xsec[0] = '\0';
+      char calicfg[1024];
+      calicfg[0] = '\0';
 
       addArg("help",             'h', 0, 'i', &(help),           0,      "print this message");
       addArg("dt",               'D', 1, 'd', &(sp.dt),          0,      "time step (seconds)");
@@ -253,12 +353,14 @@ namespace
       addArg("bTally",           'B', 1, 'i', &(sp.balanceTallyReplications), 0, "number of balance tally replications");
       addArg("fTally",           'F', 1, 'i', &(sp.fluxTallyReplications),    0, "number of scalar flux tally replications");
       addArg("cTally",           'C', 1, 'i', &(sp.cellTallyReplications),    0, "number of scalar cell tally replications");
+      addArg("caliper-config",   'P', 1, 's', &(calicfg), sizeof(calicfg), "Caliper configuration");
 
       processArgs(argc, argv);
 
       sp.inputFile = name;
       sp.energySpectrum = esName;
       sp.crossSectionsOut = xsec;
+      sp.caliperConfig = calicfg;
 
       if (help)
       {

src/Parameters.hh

@@ -164,6 +164,7 @@ struct SimulationParameters
    int fluxTallyReplications;    //!< Number of replications for the scalar flux tally
    int cellTallyReplications;    //!< Number of replications for the scalar cell tally
    int coralBenchmark;           //!< enable correctness check for Coral2 benchmark
+   std::string caliperConfig;    //!< Caliper configuration string
 };
 
 struct Parameters
@@ -176,6 +177,7 @@ struct Parameters
 
 Parameters getParameters(int argc, char** argv);
 void printParameters(const Parameters& params, std::ostream& out);
+void saveParametersInAdiak(const Parameters& parms);
 
 std::ostream& operator<<(std::ostream& out, const SimulationParameters& pp);
 std::ostream& operator<<(std::ostream& out, const GeometryParameters& pp);

src/READ.ME.HOW.TO.RUN

@@ -115,6 +115,16 @@ There is also, at the end of the run, a coarse breakdown of time spent overall
 in the above mentioned three code phases, as well as a few other sub timings
 from cycle tracking.
 
+-------------------------------------------------------------------------------
+A note on Caliper:
+
+Caliper is a powerful performance profiling/tracing library. When configured
+with Caliper support, Quicksilver adds Caliper annotations for its timed
+regions (cycleTracking, cycleTrackingKernel, etc.) as the "mc.timer"
+attribute. Performance measurements can be configured through environment
+variables or the caliper.config configuration file. For Caliper documentation,
+see https://github.com/LLNL/Caliper.
+
 -------------------------------------------------------------------------------
 A note on asserts: