Teach ExecLogParser how to parse all three execution log formats.

The format is auto-detected from the first few bytes of the input file.

PiperOrigin-RevId: 680915789
Change-Id: I6f1b4a2334912f25a8c41761a970e8f75265cc13

src/main/java/com/google/devtools/build/lib/exec/BUILD

@@ -379,7 +379,6 @@ java_library(
         ":spawn_strategy_registry",
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/protobuf:failure_details_java_proto",
-        "//third_party:flogger",
         "//third_party:guava",
     ],
 )
@@ -417,7 +416,6 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
         "//src/main/java/com/google/devtools/build/lib/events",
-        "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/util/io",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",

src/main/java/com/google/devtools/build/lib/exec/SpawnLogReconstructor.java

@@ -166,8 +166,11 @@ private SpawnExec reconstructSpawnExec(ExecLogEntry.Spawn entry) throws IOExcept
             .setRemotable(entry.getRemotable())
             .setCacheable(entry.getCacheable())
             .setRemoteCacheable(entry.getRemoteCacheable())
-            .setTimeoutMillis(entry.getTimeoutMillis())
-            .setMetrics(entry.getMetrics());
+            .setTimeoutMillis(entry.getTimeoutMillis());
+
+    if (entry.hasMetrics()) {
+      builder.setMetrics(entry.getMetrics());
+    }
 
     if (entry.hasPlatform()) {
       builder.setPlatform(entry.getPlatform());

src/tools/execlog/README.md

@@ -1,12 +1,14 @@
 # Execution Log Parser
 
-This tool is used to inspect and parse the Bazel execution logs.
+This tool is used to inspect and parse the Bazel execution logs. Currently
+supported formats are `binary`, `json`, and `compact`.
+
 To generate the execution log, run e.g.:
 
         bazel build \
-            --execution_log_binary_file=/tmp/exec.log :hello_world
+            --execution_log_compact_file=/tmp/exec.log :hello_world
 
-Then build the parser and run it.
+Then build the parser and run it:
 
         bazel build src/tools/execlog:parser
         bazel-bin/src/tools/execlog/parser --log_path=/tmp/exec.log

src/tools/execlog/src/main/java/com/google/devtools/build/execlog/BUILD

@@ -18,9 +18,12 @@ java_library(
         "ParserOptions.java",
     ],
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/exec:spawn_log_context_utils",
+        "//src/main/java/com/google/devtools/build/lib/util/io:io-proto",
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/protobuf:spawn_java_proto",
         "//third_party:guava",
+        "//third_party:jsr305",
     ],
 )
 

src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ExecLogParser.java

@@ -17,6 +17,10 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.devtools.build.lib.exec.Protos.SpawnExec;
+import com.google.devtools.build.lib.exec.SpawnLogReconstructor;
+import com.google.devtools.build.lib.util.io.MessageInputStream;
+import com.google.devtools.build.lib.util.io.MessageInputStreamWrapper.BinaryInputStreamWrapper;
+import com.google.devtools.build.lib.util.io.MessageInputStreamWrapper.JsonInputStreamWrapper;
 import com.google.devtools.common.options.OptionsParser;
 import java.io.BufferedWriter;
 import java.io.FileInputStream;
@@ -32,42 +36,78 @@
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Queue;
+import javax.annotation.Nullable;
 
-/**
- * A tool to inspect and parse the Bazel execution log.
- */
-final class ExecLogParser {
+/** A tool to inspect and parse the Bazel execution log. */
+public final class ExecLogParser {
+  private ExecLogParser() {}
 
-  static final String DELIMITER = "\n---------------------------------------------------------\n";
+  private static final String DELIMITER =
+      "\n---------------------------------------------------------\n";
+
+  private static byte[] readFirstFourBytes(String path) throws IOException {
+    try (InputStream in = new FileInputStream(path)) {
+      return in.readNBytes(4);
+    }
+  }
 
   @VisibleForTesting
-  interface Parser {
-    SpawnExec getNext() throws IOException;
+  static MessageInputStream<SpawnExec> getMessageInputStream(String path) throws IOException {
+    byte[] b = readFirstFourBytes(path);
+    if (b.length == 4
+        && b[0] == 0x28
+        && b[1] == (byte) 0xb5
+        && b[2] == 0x2f
+        && b[3] == (byte) 0xfd) {
+      // Looks like a compact file (zstd-compressed).
+      // This is definitely not a JSON file (the first byte is not '{') and definitely not a
+      // binary file (the first byte would indicate the size of the first message, and the
+      // second byte would indicate an invalid wire type).
+      return new SpawnLogReconstructor(new FileInputStream(path));
+    }
+    if (b.length >= 2 && b[0] == '{' && b[1] == '\n') {
+      // Looks like a JSON file.
+      // This is definitely not a compact file (the first byte is not 0x28) and definitely not a
+      // binary file (the first byte would indicate the size of the first message, and the
+      // second byte would indicate a field with number 1 and wire type I32, which doesn't match
+      // the proto definition).
+      return new JsonInputStreamWrapper<>(
+          new FileInputStream(path), SpawnExec.getDefaultInstance());
+    }
+    // Otherwise assume it's a binary file.
+    return new BinaryInputStreamWrapper<>(
+        new FileInputStream(path), SpawnExec.getDefaultInstance());
   }
 
   @VisibleForTesting
-  static class FilteringLogParser implements Parser {
-    final InputStream in;
+  static class FilteredStream implements MessageInputStream<SpawnExec> {
+    final MessageInputStream<SpawnExec> in;
     final String restrictToRunner;
 
-    FilteringLogParser(InputStream in, String restrictToRunner) {
+    FilteredStream(MessageInputStream<SpawnExec> in, String restrictToRunner) {
       this.in = in;
       this.restrictToRunner = restrictToRunner;
     }
 
     @Override
-    public SpawnExec getNext() throws IOException {
+    @Nullable
+    public SpawnExec read() throws IOException {
       SpawnExec ex;
-      // Find the next record whose runner matches
-      while ((ex = SpawnExec.parseDelimitedFrom(in)) != null) {
+      while ((ex = in.read()) != null) {
         if (restrictToRunner == null || restrictToRunner.equals(ex.getRunner())) {
           return ex;
         }
       }
       return null;
     }
+
+    @Override
+    public void close() throws IOException {
+      in.close();
+    }
   }
 
+  @Nullable
   static String getFirstOutput(SpawnExec e) {
     if (e.getListedOutputsCount() > 0) {
       return e.getListedOutputs(0);
@@ -76,7 +116,7 @@ static String getFirstOutput(SpawnExec e) {
   }
 
   @VisibleForTesting
-  static class ReorderingParser implements Parser {
+  static class OrderedStream implements MessageInputStream<SpawnExec> {
 
     public static class Golden {
       // A map of positions of actions in the first file.
@@ -116,24 +156,26 @@ public Element(int position, SpawnExec element) {
       }
     }
 
+    private final MessageInputStream<SpawnExec> in;
     private final Golden golden;
 
-    ReorderingParser(Golden golden, Parser input) throws IOException {
+    OrderedStream(Golden golden, MessageInputStream<SpawnExec> in) throws IOException {
+      this.in = in;
       this.golden = golden;
-      processInputFile(input);
+      processInputFile();
     }
 
     // actions from input that appear in golden, indexed by their position in the golden.
     PriorityQueue<Element> sameActions;
     // actions in input that are not in the golden, in order received.
     Queue<SpawnExec> uniqueActions;
 
-    private void processInputFile(Parser input) throws IOException {
+    private void processInputFile() throws IOException {
       sameActions = new PriorityQueue<>((e1, e2) -> (e1.position - e2.position));
       uniqueActions = new ArrayDeque<>();
 
       SpawnExec ex;
-      while ((ex = input.getNext()) != null) {
+      while ((ex = in.read()) != null) {
         int position = golden.positionFor(ex);
         if (position >= 0) {
           sameActions.add(new Element(position, ex));
@@ -144,20 +186,26 @@ private void processInputFile(Parser input) throws IOException {
     }
 
     @Override
-    public SpawnExec getNext() {
+    public SpawnExec read() {
       if (sameActions.isEmpty()) {
         return uniqueActions.poll();
       }
       return sameActions.remove().element;
     }
+
+    @Override
+    public void close() throws IOException {
+      in.close();
+    }
   }
 
-  public static void output(Parser p, OutputStream outStream, ReorderingParser.Golden golden)
+  public static void output(
+      MessageInputStream<SpawnExec> in, OutputStream outStream, OrderedStream.Golden golden)
       throws IOException {
     PrintWriter out =
         new PrintWriter(new BufferedWriter(new OutputStreamWriter(outStream, UTF_8)), true);
     SpawnExec ex;
-    while ((ex = p.getNext()) != null) {
+    while ((ex = in.read()) != null) {
       out.println(ex);
       out.println(DELIMITER);
       if (golden != null) {
@@ -211,13 +259,13 @@ public static void main(String[] args) throws Exception {
       }
     }
 
-    ReorderingParser.Golden golden = null;
+    OrderedStream.Golden golden = null;
     if (secondPath != null) {
-      golden = new ReorderingParser.Golden();
+      golden = new OrderedStream.Golden();
     }
 
-    try (InputStream input = new FileInputStream(logPath)) {
-      Parser parser = new FilteringLogParser(input, options.restrictToRunner);
+    try (MessageInputStream<SpawnExec> input = getMessageInputStream(logPath)) {
+      FilteredStream parser = new FilteredStream(input, options.restrictToRunner);
 
       if (output1 == null) {
         output(parser, System.out, golden);
@@ -229,12 +277,12 @@ public static void main(String[] args) throws Exception {
     }
 
     if (secondPath != null) {
-      try (InputStream file2 = new FileInputStream(secondPath);
+      try (MessageInputStream<SpawnExec> file2 = getMessageInputStream(secondPath);
           OutputStream output = new FileOutputStream(output2)) {
-        Parser parser = new FilteringLogParser(file2, options.restrictToRunner);
+        MessageInputStream<SpawnExec> parser = new FilteredStream(file2, options.restrictToRunner);
         // ReorderingParser will read the whole golden on initialization,
         // so it is safe to close after.
-        parser = new ReorderingParser(golden, parser);
+        parser = new OrderedStream(golden, parser);
         output(parser, output, null);
       }
     }

src/tools/execlog/test/main/java/com/google/devtools/build/execlog/BUILD

@@ -22,9 +22,12 @@ java_test(
     srcs = ["ExecLogParserTest.java"],
     test_class = "com.google.devtools.build.execlog.ExecLogParserTest",
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/exec:spawn_log_context_utils",
+        "//src/main/java/com/google/devtools/build/lib/util/io:io-proto",
         "//src/main/protobuf:spawn_java_proto",
         "//src/tools/execlog/src/main/java/com/google/devtools/build/execlog:parser",
         "//third_party:junit4",
         "//third_party:truth",
+        "@zstd-jni",
     ],
 )