[llvm-exegesis] Add support for validation counters (#76653)

This patch adds support for validation counters. Validation counters can
be used to measure events that occur during snippet execution like cache
misses to ensure that certain assumed invariants about the benchmark
actually hold. Validation counters are setup within a perf event group,
so are turned on and off at exactly the same time as the "group leader"
counter that measures the desired value.

Author: boomanaiden154

llvm/test/tools/llvm-exegesis/X86/validation-counters.asm

@@ -0,0 +1,12 @@
+# REQUIRES: exegesis-can-measure-latency, exegesis-can-measure-uops, x86_64-linux
+
+# Check that when specifying validation counters, the validation counter is
+# collected and the information is displayed in the output. Test across
+# multiple configurations that need to be wired up separately for validation
+# counter support.
+
+# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=latency -opcode-name=ADD64rr --validation-counter=instructions-retired | FileCheck %s
+# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=latency -opcode-name=ADD64rr --validation-counter=instructions-retired -execution-mode=subprocess | FileCheck %s
+# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=uops -opcode-name=ADD64rr --validation-counter=instructions-retired -execution-mode=subprocess | FileCheck %s
+
+# CHECK: instructions-retired: {{[0-9]+}}

llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp

@@ -14,6 +14,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/bit.h"
 #include "llvm/ObjectYAML/YAML.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
@@ -192,6 +193,41 @@ template <> struct SequenceElementTraits<exegesis::BenchmarkMeasure> {
   static const bool flow = false;
 };
 
+const char *validationEventToString(exegesis::ValidationEvent VE) {
+  switch (VE) {
+  case exegesis::ValidationEvent::InstructionRetired:
+    return "instructions-retired";
+  }
+}
+
+Expected<exegesis::ValidationEvent> stringToValidationEvent(StringRef Input) {
+  if (Input == "instructions-retired")
+    return exegesis::ValidationEvent::InstructionRetired;
+  else
+    return make_error<StringError>("Invalid validation event string",
+                                   errc::invalid_argument);
+}
+
+template <>
+struct CustomMappingTraits<std::map<exegesis::ValidationEvent, int64_t>> {
+  static void inputOne(IO &Io, StringRef KeyStr,
+                       std::map<exegesis::ValidationEvent, int64_t> &VI) {
+    Expected<exegesis::ValidationEvent> Key = stringToValidationEvent(KeyStr);
+    if (!Key) {
+      Io.setError("Key is not a valid validation event");
+      return;
+    }
+    Io.mapRequired(KeyStr.str().c_str(), VI[*Key]);
+  }
+
+  static void output(IO &Io, std::map<exegesis::ValidationEvent, int64_t> &VI) {
+    for (auto &IndividualVI : VI) {
+      Io.mapRequired(validationEventToString(IndividualVI.first),
+                     IndividualVI.second);
+    }
+  }
+};
+
 // exegesis::Measure is rendererd as a flow instead of a list.
 // e.g. { "key": "the key", "value": 0123 }
 template <> struct MappingTraits<exegesis::BenchmarkMeasure> {
@@ -203,6 +239,7 @@ template <> struct MappingTraits<exegesis::BenchmarkMeasure> {
     }
     Io.mapRequired("value", Obj.PerInstructionValue);
     Io.mapOptional("per_snippet_value", Obj.PerSnippetValue);
+    Io.mapOptional("validation_counters", Obj.ValidationCounters);
   }
   static const bool flow = true;
 };

llvm/tools/llvm-exegesis/lib/BenchmarkResult.h

@@ -32,6 +32,8 @@ class Error;
 
 namespace exegesis {
 
+enum ValidationEvent { InstructionRetired };
+
 enum class BenchmarkPhaseSelectorE {
   PrepareSnippet,
   PrepareAndAssembleSnippet,
@@ -77,8 +79,10 @@ struct BenchmarkKey {
 
 struct BenchmarkMeasure {
   // A helper to create an unscaled BenchmarkMeasure.
-  static BenchmarkMeasure Create(std::string Key, double Value) {
-    return {Key, Value, Value};
+  static BenchmarkMeasure
+  Create(std::string Key, double Value,
+         std::map<ValidationEvent, int64_t> ValCounters) {
+    return {Key, Value, Value, ValCounters};
   }
   std::string Key;
   // This is the per-instruction value, i.e. measured quantity scaled per
@@ -87,6 +91,8 @@ struct BenchmarkMeasure {
   // This is the per-snippet value, i.e. measured quantity for one repetition of
   // the whole snippet.
   double PerSnippetValue;
+  // These are the validation counter values.
+  std::map<ValidationEvent, int64_t> ValidationCounters;
 };
 
 // The result of an instruction benchmark.

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

@@ -54,9 +54,11 @@ namespace exegesis {
 
 BenchmarkRunner::BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
                                  BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-                                 ExecutionModeE ExecutionMode)
+                                 ExecutionModeE ExecutionMode,
+                                 ArrayRef<ValidationEvent> ValCounters)
     : State(State), Mode(Mode), BenchmarkPhaseSelector(BenchmarkPhaseSelector),
-      ExecutionMode(ExecutionMode), Scratch(std::make_unique<ScratchSpace>()) {}
+      ExecutionMode(ExecutionMode), ValidationCounters(ValCounters),
+      Scratch(std::make_unique<ScratchSpace>()) {}
 
 BenchmarkRunner::~BenchmarkRunner() = default;
 
@@ -71,16 +73,18 @@ void BenchmarkRunner::FunctionExecutor::accumulateCounterValues(
 }
 
 Expected<llvm::SmallVector<int64_t, 4>>
-BenchmarkRunner::FunctionExecutor::runAndSample(const char *Counters) const {
+BenchmarkRunner::FunctionExecutor::runAndSample(
+    const char *Counters, ArrayRef<const char *> ValidationCounters,
+    SmallVectorImpl<int64_t> &ValidationCounterValues) const {
   // We sum counts when there are several counters for a single ProcRes
   // (e.g. P23 on SandyBridge).
   llvm::SmallVector<int64_t, 4> CounterValues;
   SmallVector<StringRef, 2> CounterNames;
   StringRef(Counters).split(CounterNames, '+');
   for (auto &CounterName : CounterNames) {
     CounterName = CounterName.trim();
-    Expected<SmallVector<int64_t, 4>> ValueOrError =
-        runWithCounter(CounterName);
+    Expected<SmallVector<int64_t, 4>> ValueOrError = runWithCounter(
+        CounterName, ValidationCounters, ValidationCounterValues);
     if (!ValueOrError)
       return ValueOrError.takeError();
     accumulateCounterValues(ValueOrError.get(), &CounterValues);
@@ -120,11 +124,13 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
       (*Result)[I] += NewValues[I];
   }
 
-  Expected<llvm::SmallVector<int64_t, 4>>
-  runWithCounter(StringRef CounterName) const override {
+  Expected<llvm::SmallVector<int64_t, 4>> runWithCounter(
+      StringRef CounterName, ArrayRef<const char *> ValidationCounters,
+      SmallVectorImpl<int64_t> &ValidationCounterValues) const override {
     const ExegesisTarget &ET = State.getExegesisTarget();
     char *const ScratchPtr = Scratch->ptr();
-    auto CounterOrError = ET.createCounter(CounterName, State);
+    auto CounterOrError =
+        ET.createCounter(CounterName, State, ValidationCounters);
 
     if (!CounterOrError)
       return CounterOrError.takeError();
@@ -156,6 +162,14 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
       }
     }
 
+    auto ValidationValuesOrErr = Counter->readValidationCountersOrError();
+    if (!ValidationValuesOrErr)
+      return ValidationValuesOrErr.takeError();
+
+    ArrayRef RealValidationValues = *ValidationValuesOrErr;
+    for (size_t I = 0; I < RealValidationValues.size(); ++I)
+      ValidationCounterValues[I] = RealValidationValues[I];
+
     return Counter->readOrError(Function.getFunctionBytes());
   }
 
@@ -266,7 +280,9 @@ class SubProcessFunctionExecutorImpl
   }
 
   Error createSubProcessAndRunBenchmark(
-      StringRef CounterName, SmallVectorImpl<int64_t> &CounterValues) const {
+      StringRef CounterName, SmallVectorImpl<int64_t> &CounterValues,
+      ArrayRef<const char *> ValidationCounters,
+      SmallVectorImpl<int64_t> &ValidationCounterValues) const {
     int PipeFiles[2];
     int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles);
     if (PipeSuccessOrErr != 0) {
@@ -306,8 +322,8 @@ class SubProcessFunctionExecutorImpl
     }
 
     const ExegesisTarget &ET = State.getExegesisTarget();
-    auto CounterOrError =
-        ET.createCounter(CounterName, State, ParentOrChildPID);
+    auto CounterOrError = ET.createCounter(
+        CounterName, State, ValidationCounters, ParentOrChildPID);
 
     if (!CounterOrError)
       return CounterOrError.takeError();
@@ -362,6 +378,14 @@ class SubProcessFunctionExecutorImpl
           return CounterValueOrErr.takeError();
         CounterValues = std::move(*CounterValueOrErr);
 
+        auto ValidationValuesOrErr = Counter->readValidationCountersOrError();
+        if (!ValidationValuesOrErr)
+          return ValidationValuesOrErr.takeError();
+
+        ArrayRef RealValidationValues = *ValidationValuesOrErr;
+        for (size_t I = 0; I < RealValidationValues.size(); ++I)
+          ValidationCounterValues[I] = RealValidationValues[I];
+
         return Error::success();
       }
       // The child exited, but not successfully
@@ -460,15 +484,15 @@ class SubProcessFunctionExecutorImpl
     exit(0);
   }
 
-  Expected<llvm::SmallVector<int64_t, 4>>
-  runWithCounter(StringRef CounterName) const override {
+  Expected<llvm::SmallVector<int64_t, 4>> runWithCounter(
+      StringRef CounterName, ArrayRef<const char *> ValidationCounters,
+      SmallVectorImpl<int64_t> &ValidationCounterValues) const override {
     SmallVector<int64_t, 4> Value(1, 0);
-    Error PossibleBenchmarkError =
-        createSubProcessAndRunBenchmark(CounterName, Value);
+    Error PossibleBenchmarkError = createSubProcessAndRunBenchmark(
+        CounterName, Value, ValidationCounters, ValidationCounterValues);
 
-    if (PossibleBenchmarkError) {
+    if (PossibleBenchmarkError)
       return std::move(PossibleBenchmarkError);
-    }
 
     return Value;
   }
@@ -642,6 +666,34 @@ BenchmarkRunner::writeObjectFile(StringRef Buffer, StringRef FileName) const {
   return std::string(ResultPath);
 }
 
+static bool EventLessThan(const std::pair<ValidationEvent, const char *> LHS,
+                          const ValidationEvent RHS) {
+  return static_cast<int>(LHS.first) < static_cast<int>(RHS);
+}
+
+Error BenchmarkRunner::getValidationCountersToRun(
+    SmallVector<const char *> &ValCountersToRun) const {
+  const PfmCountersInfo &PCI = State.getPfmCounters();
+  ValCountersToRun.reserve(ValidationCounters.size());
+
+  ValCountersToRun.reserve(ValidationCounters.size());
+  ArrayRef TargetValidationEvents(PCI.ValidationEvents,
+                                  PCI.NumValidationEvents);
+  for (const ValidationEvent RequestedValEvent : ValidationCounters) {
+    auto ValCounterIt =
+        lower_bound(TargetValidationEvents, RequestedValEvent, EventLessThan);
+    if (ValCounterIt == TargetValidationEvents.end() ||
+        ValCounterIt->first != RequestedValEvent)
+      return make_error<Failure>("Cannot create validation counter");
+
+    assert(ValCounterIt->first == RequestedValEvent &&
+           "The array of validation events from the target should be sorted");
+    ValCountersToRun.push_back(ValCounterIt->second);
+  }
+
+  return Error::success();
+}
+
 BenchmarkRunner::FunctionExecutor::~FunctionExecutor() {}
 
 } // namespace exegesis

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h

@@ -38,7 +38,8 @@ class BenchmarkRunner {
 
   explicit BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
                            BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-                           ExecutionModeE ExecutionMode);
+                           ExecutionModeE ExecutionMode,
+                           ArrayRef<ValidationEvent> ValCounters);
 
   virtual ~BenchmarkRunner();
 
@@ -93,14 +94,18 @@ class BenchmarkRunner {
     virtual ~FunctionExecutor();
 
     Expected<llvm::SmallVector<int64_t, 4>>
-    runAndSample(const char *Counters) const;
+    runAndSample(const char *Counters,
+                 ArrayRef<const char *> ValidationCounters,
+                 SmallVectorImpl<int64_t> &ValidationCounterValues) const;
 
   protected:
     static void
     accumulateCounterValues(const llvm::SmallVectorImpl<int64_t> &NewValues,
                             llvm::SmallVectorImpl<int64_t> *Result);
     virtual Expected<llvm::SmallVector<int64_t, 4>>
-    runWithCounter(StringRef CounterName) const = 0;
+    runWithCounter(StringRef CounterName,
+                   ArrayRef<const char *> ValidationCounters,
+                   SmallVectorImpl<int64_t> &ValidationCounterValues) const = 0;
   };
 
 protected:
@@ -109,6 +114,11 @@ class BenchmarkRunner {
   const BenchmarkPhaseSelectorE BenchmarkPhaseSelector;
   const ExecutionModeE ExecutionMode;
 
+  SmallVector<ValidationEvent> ValidationCounters;
+
+  Error
+  getValidationCountersToRun(SmallVector<const char *> &ValCountersToRun) const;
+
 private:
   virtual Expected<std::vector<BenchmarkMeasure>>
   runMeasurements(const FunctionExecutor &Executor) const = 0;

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp

@@ -22,8 +22,9 @@ LatencyBenchmarkRunner::LatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
     Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode,
-    unsigned BenchmarkRepeatCount)
-    : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) {
+    ArrayRef<ValidationEvent> ValCounters, unsigned BenchmarkRepeatCount)
+    : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode,
+                      ValCounters) {
   assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) &&
          "invalid mode");
   ResultAggMode = ResultAgg;
@@ -72,11 +73,21 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
   // ResultAggMode.
   llvm::SmallVector<int64_t, 4> AccumulatedValues;
   double MinVariance = std::numeric_limits<double>::infinity();
-  const char *CounterName = State.getPfmCounters().CycleCounter;
+  const PfmCountersInfo &PCI = State.getPfmCounters();
+  const char *CounterName = PCI.CycleCounter;
+
+  SmallVector<const char *> ValCountersToRun;
+  Error ValCounterErr = getValidationCountersToRun(ValCountersToRun);
+  if (ValCounterErr)
+    return std::move(ValCounterErr);
+
+  SmallVector<int64_t> ValCounterValues(ValCountersToRun.size(), 0);
   // Values count for each run.
   int ValuesCount = 0;
   for (size_t I = 0; I < NumMeasurements; ++I) {
-    auto ExpectedCounterValues = Executor.runAndSample(CounterName);
+    SmallVector<int64_t> IterationValCounterValues(ValCountersToRun.size(), -1);
+    auto ExpectedCounterValues = Executor.runAndSample(
+        CounterName, ValCountersToRun, IterationValCounterValues);
     if (!ExpectedCounterValues)
       return ExpectedCounterValues.takeError();
     ValuesCount = ExpectedCounterValues.get().size();
@@ -90,8 +101,15 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
         MinVariance = Variance;
       }
     }
+
+    for (size_t I = 0; I < ValCounterValues.size(); ++I)
+      ValCounterValues[I] += IterationValCounterValues[I];
   }
 
+  std::map<ValidationEvent, int64_t> ValidationInfo;
+  for (size_t I = 0; I < ValidationCounters.size(); ++I)
+    ValidationInfo[ValidationCounters[I]] = ValCounterValues[I];
+
   std::string ModeName;
   switch (Mode) {
   case Benchmark::Latency:
@@ -112,25 +130,26 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
     std::vector<BenchmarkMeasure> Result;
     Result.reserve(AccumulatedValues.size());
     for (const int64_t Value : AccumulatedValues)
-      Result.push_back(BenchmarkMeasure::Create(ModeName, Value));
+      Result.push_back(
+          BenchmarkMeasure::Create(ModeName, Value, ValidationInfo));
     return std::move(Result);
   }
   case Benchmark::Min: {
     std::vector<BenchmarkMeasure> Result;
-    Result.push_back(
-        BenchmarkMeasure::Create(ModeName, findMin(AccumulatedValues)));
+    Result.push_back(BenchmarkMeasure::Create(
+        ModeName, findMin(AccumulatedValues), ValidationInfo));
     return std::move(Result);
   }
   case Benchmark::Max: {
     std::vector<BenchmarkMeasure> Result;
-    Result.push_back(
-        BenchmarkMeasure::Create(ModeName, findMax(AccumulatedValues)));
+    Result.push_back(BenchmarkMeasure::Create(
+        ModeName, findMax(AccumulatedValues), ValidationInfo));
     return std::move(Result);
   }
   case Benchmark::Mean: {
     std::vector<BenchmarkMeasure> Result;
-    Result.push_back(
-        BenchmarkMeasure::Create(ModeName, findMean(AccumulatedValues)));
+    Result.push_back(BenchmarkMeasure::Create(
+        ModeName, findMean(AccumulatedValues), ValidationInfo));
     return std::move(Result);
   }
   }