#2302: Update interface to PAPI metrics

Author: pierrepebay

ci/build_cpp.sh

@@ -80,8 +80,10 @@ else
     cd "${source_dir}/lib"
     git clone https://github.com/icl-utk-edu/papi.git
     cd papi/src
-    ./configure --prefix=${source_dir}/lib/papi/install
-    make && make install
+    export PAPI_BUILD=${build_dir}/papi
+    mkdir -p "$PAPI_BUILD"
+    CC="${CC:-cc}" F77="${F77:-gfortran}" ./configure --prefix=${PAPI_BUILD}/install
+    make -j ${dashj} && make install
 fi
 
 if test "${VT_ZOLTAN_ENABLED:-0}" -eq 1
@@ -132,6 +134,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
       -DCMAKE_C_COMPILER="${CC:-cc}" \
       -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS:-}" \
       -Dcheckpoint_ROOT="$CHECKPOINT_BUILD/install" \
+      -Dpapi_ROOT="${build_dir}/papi/install" \
       -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH:-}" \
       -DCMAKE_INSTALL_PREFIX="$VT_BUILD/install" \
       -Dvt_ci_build="${VT_CI_BUILD:-0}" \

cmake/FindPAPI.cmake

@@ -1,11 +1,11 @@
 # Set minimum CMake version
 cmake_minimum_required(VERSION 3.23 FATAL_ERROR)
 
-set(PAPI_ROOT "${CMAKE_SOURCE_DIR}/lib/papi/install")
 
 # Find the PAPI include directory and library
-find_path(PAPI_INCLUDE_DIR NAMES papi.h HINTS ${PAPI_ROOT}/include)
-find_library(PAPI_LIBRARY NAMES papi HINTS ${PAPI_ROOT}/lib)
+message(STATUS "Finding PAPI, PAPI_BUILD_ROOT: ${papi_ROOT}")
+find_path(PAPI_INCLUDE_DIR NAMES papi.h HINTS ${papi_ROOT}/include)
+find_library(PAPI_LIBRARY NAMES papi HINTS ${papi_ROOT}/lib)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(PAPI DEFAULT_MSG PAPI_LIBRARY PAPI_INCLUDE_DIR)

cmake/link_vt.cmake

@@ -216,9 +216,14 @@ function(link_target_with_vt)
     if (${ARG_DEBUG_LINK})
       message(STATUS "link_target_with_vt: papi=${ARG_LINK_PAPI}")
     endif()
+    message(STATUS "target_link_libraries: ${ARG_TARGET} PUBLIC  ${ARG_BUILD_TYPE} ${PAPI_LIBRARY}")
+    message(STATUS "target_include_directories: ${ARG_TARGET} PUBLIC  ${ARG_BUILD_TYPE} ${PAPI_INCLUDE_DIR}")
     target_link_libraries(
       ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${PAPI_LIBRARY}
     )
+    target_include_directories(
+      ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${PAPI_INCLUDE_DIR}
+    )
   endif()
 
   if (${vt_mimalloc_enabled})

src/vt/context/runnable_context/lb_data.cc

@@ -53,17 +53,15 @@ void LBData::start(TimeType time) {
   }
 
   /* -- PAPI START -- */
-
   /* Start counting events in the Event Set */
-  // papi_retval_ = PAPI_start(EventSet_);
-  // if (papi_retval_ != PAPI_OK)
-  //   handle_papi_error(papi_retval_, "LBData start: Starting counting events in the Event Set: ");
-
-  /* Gets the starting time in clock cycles */
-  start_cycles_ = PAPI_get_real_cyc();
+  papi_retval_ = PAPI_start(EventSet_);
+  if (papi_retval_ != PAPI_OK)
+    handle_papi_error(papi_retval_, "LBData start: Starting counting events in the Event Set: ");
 
-  /* Gets the starting time in microseconds */
-  start_usec_ = PAPI_get_real_usec();
+  start_real_cycles_ = PAPI_get_real_cyc();
+  start_real_usec_ = PAPI_get_real_usec();
+  start_virt_cycles_ = PAPI_get_virt_cyc();
+  start_virt_usec_ = PAPI_get_virt_usec();
 
   /* ---------------- */
 }
@@ -74,30 +72,18 @@ void LBData::finish(TimeType time) {
     lb_data_->stop(time);
   }
 
-  /* -- PAPI READ AND STOP -- */
-
-  /* Read the counting events in the Event Set */
-  // papi_retval_ = PAPI_read(EventSet_, papi_values_);
-  // if (papi_retval_ != PAPI_OK)
-  //   handle_papi_error(papi_retval_, "LBData finish: Reading the counting events in the Event Set: ");
-
-  // printf("Counters after LBData::finish: %lld\n",papi_values_[0]);
-
-  // /* Stop the counting of events in the Event Set */
-  // papi_retval_ = PAPI_stop(EventSet_, papi_values_);
-  // if (papi_retval_ != PAPI_OK)
-  //   handle_papi_error(papi_retval_, "LBData finish: Stoping the counting of events in the Event Set: ");
+  /* -- PAPI STOP -- */
+  /* Stop the counting of events in the Event Set */
+  papi_retval_ = PAPI_stop(EventSet_, papi_values_);
+  if (papi_retval_ != PAPI_OK)
+    handle_papi_error(papi_retval_, "LBData finish: Stoping the counting of events in the Event Set: ");
 
-  /* Gets the ending time in clock cycles */
-  end_cycles_ = PAPI_get_real_cyc();
+  end_real_cycles_ = PAPI_get_real_cyc();
+  end_real_usec_ = PAPI_get_real_usec();
+  end_virt_cycles_ = PAPI_get_virt_cyc();
+  end_virt_usec_ = PAPI_get_virt_usec();
 
-  /* Gets the ending time in microseconds */
-  end_usec_ = PAPI_get_real_usec();
-
-  printf("Wall clock cycles: %lld\n", end_cycles_ - start_cycles_);
-  printf("Wall clock time in microseconds: %lld\n", end_usec_ - start_usec_);
-
-  /* ------------------------ */
+  /* -------------- */
 }
 
 void LBData::send(elm::ElementIDStruct dest, MsgSizeType bytes) {
@@ -118,4 +104,12 @@ typename LBData::ElementIDStruct const& LBData::getCurrentElementID() const {
   return cur_elm_id_;
 }
 
+std::unordered_map<std::string, double> LBData::getPAPIMetrics() const {
+  std::unordered_map<std::string, double> papi_metrics = {};
+  for (size_t i = 0; i < native_events_.size(); i++) {
+    papi_metrics[native_events_[i]] = papi_values_[i];
+  }
+  return papi_metrics;
+}
+
 }} /* end namespace vt::ctx */

src/vt/context/runnable_context/lb_data.h

@@ -90,13 +90,24 @@ struct LBData {
   {
     /* Create the PAPI Event Set */
     papi_retval_ = PAPI_create_eventset(&EventSet_);
-    if (papi_retval_ != PAPI_OK)
-      handle_papi_error(papi_retval_, "LBData Constructor 2: Creating the PAPI Event Set: ");
-
-    // /* Add Total Instructions Executed to the PAPI Event Set */
-    // papi_retval_ = PAPI_add_event(EventSet_, PAPI_DP_OPS);
-    // if (papi_retval_ != PAPI_OK)
-    //   handle_papi_error(papi_retval_, "LBData Constructor 2: Adding Total Instructions Executed to the PAPI Event Set: ");
+    if (papi_retval_ != PAPI_OK) {
+      printf("LBData Constructor 2: Creating the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_));
+      exit(1);
+    }
+
+    for (const auto& event_name : native_events_) {
+      int native = 0x0;
+      papi_retval_ = PAPI_event_name_to_code(event_name.c_str(), &native);
+      if (papi_retval_ != PAPI_OK) {
+        printf("LBData Constructor 2: Couldn't event_name_to_code for %s: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_));
+        exit(1);
+      }
+      papi_retval_ = PAPI_add_event(EventSet_, native);
+      if (papi_retval_ != PAPI_OK) {
+        printf("LBData Constructor 2: Couldn't add %s to the PAPI Event Set: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_));
+        exit(1);
+      }
+    }
   }
 
   /**
@@ -133,14 +144,23 @@ struct LBData {
    */
   ElementIDStruct const& getCurrentElementID() const;
 
+  /**
+   * \brief Get the current PAPI metrics map for the running context
+   *
+   * \return the PAPI metrics map
+   */
+  std::unordered_map<std::string, double> getPAPIMetrics() const;
+
 private:
   ElementLBData* lb_data_ = nullptr;     /**< Element LB data */
   ElementIDStruct cur_elm_id_ = {};   /**< Current element ID  */
   bool should_instrument_ = false;    /**< Whether we are instrumenting */
-  long_long papi_values_[1];
   int EventSet_ = PAPI_NULL;
   int papi_retval_;
-  long long start_cycles_, end_cycles_, start_usec_, end_usec_;
+  long long start_real_cycles_, end_real_cycles_, start_real_usec_, end_real_usec_;
+  long long start_virt_cycles_, end_virt_cycles_, start_virt_usec_, end_virt_usec_;
+  std::vector<std::string> native_events_ = {"instructions", "cache-misses", "fp_arith_inst_retired.scalar_double"};
+  long_long papi_values_[3];
 };
 
 }} /* end namespace vt::ctx */

src/vt/context/runnable_context/lb_data.impl.h

@@ -63,19 +63,26 @@ LBData::LBData(ElmT* in_elm, MsgT* msg)
   // record the communication LB data right away!
   theCollection()->recordLBData(in_elm, msg);
 
-  // /* Create the PAPI Event Set */
-  // papi_retval_ = PAPI_create_eventset(&EventSet_);
-  // if (papi_retval_ != PAPI_OK) {
-  //   printf("LBData Constructor 1: Creating the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_));
-  //   exit(1);
-  // }
+  /* Create the PAPI Event Set */
+  papi_retval_ = PAPI_create_eventset(&EventSet_);
+  if (papi_retval_ != PAPI_OK) {
+    printf("LBData Constructor 1: Creating the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_));
+    exit(1);
+  }
 
-  // /* Add Total Instructions Executed to the PAPI Event Set */
-  // papi_retval_ = PAPI_add_event(EventSet_, PAPI_TOT_INS);
-  // if (papi_retval_ != PAPI_OK) {
-  //   printf("LBData Constructor 1: Adding Total Instructions Executed to the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_));
-  //   exit(1);
-  // }
+  for (const auto& event_name : native_events_) {
+    int native = 0x0;
+    papi_retval_ = PAPI_event_name_to_code(event_name.c_str(), &native);
+    if (papi_retval_ != PAPI_OK) {
+      printf("LBData Constructor 1: Couldn't event_name_to_code for %s: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_));
+      exit(1);
+    }
+    papi_retval_ = PAPI_add_event(EventSet_, native);
+    if (papi_retval_ != PAPI_OK) {
+      printf("LBData Constructor 1: Couldn't add %s to the PAPI Event Set: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_));
+      exit(1);
+    }
+  }
 }
 
 }} /* end namespace vt::ctx */

src/vt/runnable/runnable.cc

@@ -199,6 +199,15 @@ void RunnableNew::run() {
 #endif
 }
 
+std::unordered_map<std::string, double> RunnableNew::getPAPIMetrics() const {
+  std::unordered_map<std::string, double> result = {};
+  if (contexts_.has_lb)
+  {
+    result = contexts_.lb.getPAPIMetrics();
+  }
+  return result;
+}
+
 void RunnableNew::start(TimeType time) {
   contexts_.setcontext.start();
   if (contexts_.has_td) contexts_.td.start();

src/vt/runnable/runnable.h

@@ -321,6 +321,13 @@ struct RunnableNew {
    */
   BaseMsgType* getMsg() const { return msg_.get(); }
 
+  /**
+   * \brief Get the dictionnary of PAPI metrics associated with the runnable
+   *
+   * \return the dictionnary
+   */
+  std::unordered_map<std::string, double> getPAPIMetrics() const;
+
 #if vt_check_enabled(fcontext)
   /**
    * \brief Check if this runnable is complete or not