Add action stats to the BEP.

PiperOrigin-RevId: 373184320

src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto

@@ -706,6 +706,24 @@ message BuildMetrics {
     // The total number of actions executed during the build. This includes any
     // remote cache hits, but excludes local action cache hits.
     int64 actions_executed = 2;
+
+    message ActionData {
+      string mnemonic = 1;
+
+      // The total number of actions of this type executed during the build. As
+      // above, includes remote cache hits but excludes local action cache hits.
+      int64 actions_executed = 2;
+
+      // When the first action of this type started being executed, in
+      // milliseconds from the epoch.
+      int64 first_started_ms = 3;
+
+      // When the last action of this type ended being executed, in
+      // milliseconds from the epoch.
+      int64 last_ended_ms = 4;
+    }
+    // Contains the top N actions by number of actions executed.
+    repeated ActionData action_data = 4;
   }
   ActionSummary action_summary = 1;
 

src/main/java/com/google/devtools/build/lib/metrics/BUILD

@@ -37,6 +37,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_phase_complete_event",
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_phase_started_event",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
+        "//src/main/java/com/google/devtools/build/lib/clock",
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/skyframe:execution_finished_event",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe_graph_stats_event",

src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java

@@ -21,6 +21,7 @@
 import com.google.devtools.build.lib.analysis.AnalysisPhaseStartedEvent;
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics;
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.ActionSummary;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.ActionSummary.ActionData;
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.ArtifactMetrics;
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.BuildGraphMetrics;
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.CumulativeMetrics;
@@ -30,6 +31,8 @@
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.TimingMetrics;
 import com.google.devtools.build.lib.buildtool.BuildPrecompleteEvent;
 import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
+import com.google.devtools.build.lib.clock.BlazeClock;
+import com.google.devtools.build.lib.clock.BlazeClock.NanosToMillisSinceEpochConverter;
 import com.google.devtools.build.lib.metrics.MetricsModule.Options;
 import com.google.devtools.build.lib.metrics.PostGCMemoryUseRecorder.PeakHeap;
 import com.google.devtools.build.lib.profiler.Profiler;
@@ -39,14 +42,34 @@
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.time.Duration;
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAccumulator;
+import java.util.stream.Stream;
+
+class ActionStats {
+  LongAccumulator firstStarted;
+  LongAccumulator lastEnded;
+  AtomicLong numActions;
+  String mnemonic;
+
+  ActionStats(String mnemonic) {
+    this.mnemonic = mnemonic;
+    firstStarted = new LongAccumulator(Math::min, Long.MAX_VALUE);
+    lastEnded = new LongAccumulator(Math::max, 0);
+    numActions = new AtomicLong();
+  }
+}
 
 class MetricsCollector {
   private final CommandEnvironment env;
   private final boolean bepPublishUsedHeapSizePostBuild;
+  private final boolean recordMetricsForAllMnemonics;
   // For ActionSummary.
   private final AtomicLong executedActionCount = new AtomicLong();
+  private final ConcurrentHashMap<String, ActionStats> actionStatsMap = new ConcurrentHashMap<>();
 
   // For CumulativeMetrics.
   private final AtomicInteger numAnalyses;
@@ -65,6 +88,7 @@ private MetricsCollector(
     Options options = env.getOptions().getOptions(Options.class);
     this.bepPublishUsedHeapSizePostBuild =
         options != null && options.bepPublishUsedHeapSizePostBuild;
+    this.recordMetricsForAllMnemonics = options != null && options.recordMetricsForAllMnemonics;
     this.numAnalyses = numAnalyses;
     this.numBuilds = numBuilds;
     env.getEventBus().register(this);
@@ -112,6 +136,11 @@ public synchronized void logExecutionStartingEvent(ExecutionStartingEvent event)
   @AllowConcurrentEvents
   public void onActionComplete(ActionCompletionEvent event) {
     executedActionCount.incrementAndGet();
+    ActionStats actionStats =
+        actionStatsMap.computeIfAbsent(event.getAction().getMnemonic(), ActionStats::new);
+    actionStats.numActions.incrementAndGet();
+    actionStats.firstStarted.accumulate(event.getRelativeActionStartTime());
+    actionStats.lastEnded.accumulate(BlazeClock.nanoTime());
   }
 
   @SuppressWarnings("unused")
@@ -149,7 +178,31 @@ private BuildMetrics createBuildMetrics() {
         .build();
   }
 
+  private static final int MAX_ACTION_DATA = 20;
+
   private ActionSummary finishActionSummary() {
+    NanosToMillisSinceEpochConverter nanosToMillisSinceEpochConverter =
+        BlazeClock.createNanosToMillisSinceEpochConverter();
+    Stream<ActionStats> actionStatsStream = actionStatsMap.values().stream();
+    if (!recordMetricsForAllMnemonics) {
+      actionStatsStream =
+          actionStatsStream
+              .sorted(Comparator.comparingLong(a -> -a.numActions.get()))
+              .limit(MAX_ACTION_DATA);
+    }
+    actionStatsStream.forEach(
+        action ->
+            actionSummary.addActionData(
+                ActionData.newBuilder()
+                    .setMnemonic(action.mnemonic)
+                    .setFirstStartedMs(
+                        nanosToMillisSinceEpochConverter.toEpochMillis(
+                            action.firstStarted.longValue()))
+                    .setLastEndedMs(
+                        nanosToMillisSinceEpochConverter.toEpochMillis(
+                            action.lastEnded.longValue()))
+                    .setActionsExecuted(action.numActions.get())
+                    .build()));
     return actionSummary.setActionsExecuted(executedActionCount.get()).build();
   }
 

src/main/java/com/google/devtools/build/lib/metrics/MetricsModule.java

@@ -40,6 +40,17 @@ public static final class Options extends OptionsBase {
             "When set we collect and publish used_heap_size_post_build "
                 + "from build_event_stream.proto. This forces a full GC and is off by default.")
     public boolean bepPublishUsedHeapSizePostBuild;
+
+    @Option(
+        name = "experimental_record_metrics_for_all_mnemonics",
+        defaultValue = "false",
+        documentationCategory = OptionDocumentationCategory.LOGGING,
+        effectTags = {OptionEffectTag.UNKNOWN},
+        help =
+            "By default the number of action types is limited to the 20 mnemonics with the largest "
+                + "number of executed actions. Setting this option will write statistics for all "
+                + "mnemonics.")
+    public boolean recordMetricsForAllMnemonics;
   }
 
   private final AtomicInteger numAnalyses = new AtomicInteger();