Stream HTML test report processing to fix OOM on large reports

gradle-witchcraft-logging/src/main/java/com/palantir/witchcraft/java/logging/gradle/testreport/HtmlReportPostProcessor.java

@@ -16,34 +16,24 @@
 
 package com.palantir.witchcraft.java.logging.gradle.testreport;
 
-import com.palantir.witchcraft.java.logging.format.LogFormatter;
-import com.palantir.witchcraft.java.logging.format.LogParser;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
-import org.apache.commons.text.StringEscapeUtils;
 
 /**
  * Post-processes HTML test report files to apply witchcraft log formatting.
+ * Streams line-by-line to support arbitrarily large files in constant memory.
  */
 final class HtmlReportPostProcessor {
 
-    private static final LogParser<Optional<String>> PARSER = new LogParser<>(TestLogFilter.INSTANCE.combineWith(
-            LogFormatter.INSTANCE, (include, formatted) -> include ? Optional.of(formatted) : Optional.empty()));
-
-    // Match <pre> tags within stdout/stderr sections: <h2>standard output</h2>...<pre>...</pre>
-    private static final Pattern OUTPUT_SECTION_PATTERN = Pattern.compile(
-            "(<h2>standard (?:output|error)</h2>\\s*<span[^>]*>\\s*<pre[^>]*>)(.*?)(</pre>)",
-            Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
-
     void processReportDirectory(File reportDir) {
         Optional.ofNullable(reportDir).filter(File::isDirectory).ifPresent(this::walkAndProcessHtmlFiles);
     }
@@ -57,45 +47,14 @@ private void walkAndProcessHtmlFiles(File reportDir) {
     }
 
     private void processHtmlFile(Path htmlFile) {
-        try {
-            String content = Files.readString(htmlFile, StandardCharsets.UTF_8);
-            String processed = processHtmlContent(content);
-
-            if (!content.equals(processed)) {
-                Files.writeString(htmlFile, processed, StandardCharsets.UTF_8);
-            }
+        Path tmp = htmlFile.resolveSibling(htmlFile.getFileName() + ".tmp");
+        try (BufferedReader reader = Files.newBufferedReader(htmlFile, StandardCharsets.UTF_8);
+                BufferedWriter writer = Files.newBufferedWriter(tmp, StandardCharsets.UTF_8)) {
+            StreamState state = new StreamState(writer);
+            reader.lines().forEach(state::processLine);
+            Files.move(tmp, htmlFile, StandardCopyOption.REPLACE_EXISTING);
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
     }
-
-    private String processHtmlContent(String html) {
-        Matcher matcher = OUTPUT_SECTION_PATTERN.matcher(html);
-        StringBuilder result = new StringBuilder();
-
-        while (matcher.find()) {
-            String replacement = matcher.group(1) + formatPreContent(matcher.group(2)) + matcher.group(3);
-            matcher.appendReplacement(result, Matcher.quoteReplacement(replacement));
-        }
-        matcher.appendTail(result);
-
-        return result.toString();
-    }
-
-    private String formatPreContent(String content) {
-        return content.lines()
-                .map(this::formatLine)
-                .<String>mapMulti(Optional::ifPresent)
-                .collect(Collectors.joining("\n"));
-    }
-
-    /**
-     * @return formatted line (with trailing newline), empty if filtered out,
-     * or original if not a witchcraft log
-     */
-    private Optional<String> formatLine(String line) {
-        return PARSER.tryParse(StringEscapeUtils.unescapeHtml4(line))
-                .map(opt -> opt.map(formatted -> StringEscapeUtils.escapeHtml4(formatted) + "\n"))
-                .orElseGet(() -> Optional.of(line));
-    }
 }

gradle-witchcraft-logging/src/main/java/com/palantir/witchcraft/java/logging/gradle/testreport/StreamState.java

@@ -0,0 +1,152 @@
+/*
+ * (c) Copyright 2026 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.witchcraft.java.logging.gradle.testreport;
+
+import com.palantir.witchcraft.java.logging.format.LogFormatter;
+import com.palantir.witchcraft.java.logging.format.LogParser;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.text.StringEscapeUtils;
+
+/// A state machine that processes Gradle HTML test report lines, filtering and
+/// formatting log output within `<pre>` blocks that follow standard
+/// output/error headers.
+///
+/// ```
+///  ┌───────────┐  <h2>   ┌──────────────┐ <pre> ┌────────┐
+///  │  DEFAULT  │────────▶│ AWAITING_PRE │──────▶│ IN_PRE │
+///  └─────┬─────┘         └──────────────┘       └───┬────┘
+///    ↺ any line            ↺ any line            ↺ any line
+///    passthrough            passthrough           formatAndWrite
+///        ▲                                          │
+///        └──────── </pre> ── formatAndWrite ────────┘
+/// ```
+final class StreamState {
+    private final LineWriter lineWriter;
+    private State state = Default.INSTANCE;
+
+    StreamState(BufferedWriter writer) {
+        this.lineWriter = new LineWriter(writer);
+    }
+
+    void processLine(String line) {
+        try {
+            StepResult result = state.step(lineWriter, line);
+            state = result.next();
+            if (result.remainder().isPresent()) {
+                result = state.step(lineWriter, result.remainder().get());
+                state = result.next();
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static final class Default implements State {
+        static final State INSTANCE = new Default();
+
+        private static final Pattern OUTPUT_HEADER_PATTERN =
+                Pattern.compile("<h2>standard (?:output|error)</h2>", Pattern.CASE_INSENSITIVE);
+
+        @Override
+        public StepResult step(LineWriter writer, String input) throws IOException {
+            writer.writeLine(input);
+            if (OUTPUT_HEADER_PATTERN.matcher(input).find()) {
+                return StepResult.of(AwaitingPre.INSTANCE);
+            }
+            return StepResult.of(this);
+        }
+    }
+
+    private static final class AwaitingPre implements State {
+        static final State INSTANCE = new AwaitingPre();
+
+        private static final Pattern PRE_OPEN_PATTERN = Pattern.compile("<pre[^>]*>");
+
+        @Override
+        public StepResult step(LineWriter writer, String input) throws IOException {
+            Matcher matcher = PRE_OPEN_PATTERN.matcher(input);
+            if (!matcher.find()) {
+                writer.writeLine(input);
+                return StepResult.of(this);
+            }
+
+            writer.writeLine(input.substring(0, matcher.end()));
+
+            return StepResult.of(input.substring(matcher.end()));
+        }
+    }
+
+    private static final class InPre implements State {
+        private static final LogParser<Optional<String>> PARSER = new LogParser<>(TestLogFilter.INSTANCE.combineWith(
+                LogFormatter.INSTANCE, (include, formatted) -> include ? Optional.of(formatted) : Optional.empty()));
+
+        static final State INSTANCE = new InPre();
+
+        @Override
+        public StepResult step(LineWriter writer, String input) throws IOException {
+            int closeIdx = input.indexOf("</pre>");
+            if (closeIdx < 0) {
+                Optional<String> formatted = formatLine(input);
+                if (formatted.isPresent()) {
+                    writer.writeLine(formatted.get());
+                }
+                return StepResult.of(this);
+            }
+
+            if (closeIdx > 0) {
+                Optional<String> formatted = formatLine(input.substring(0, closeIdx));
+                if (formatted.isPresent()) {
+                    writer.writeLine(formatted.get());
+                }
+            }
+            writer.writeLine(input.substring(closeIdx));
+            return StepResult.of(Default.INSTANCE);
+        }
+
+        private static Optional<String> formatLine(String line) {
+            return PARSER.tryParse(StringEscapeUtils.unescapeHtml4(line))
+                    .map(opt -> opt.map(StringEscapeUtils::escapeHtml4))
+                    .orElseGet(() -> Optional.of(line));
+        }
+    }
+
+    private record LineWriter(BufferedWriter writer) {
+        void writeLine(String line) throws IOException {
+            writer.write(line);
+            writer.newLine();
+        }
+    }
+
+    private record StepResult(Optional<String> remainder, State next) {
+        static StepResult of(State next) {
+            return new StepResult(Optional.empty(), next);
+        }
+
+        static StepResult of(String remainder) {
+            return new StepResult(Optional.of(remainder), InPre.INSTANCE);
+        }
+    }
+
+    private sealed interface State {
+        StepResult step(LineWriter writer, String input) throws IOException;
+    }
+}

gradle-witchcraft-logging/src/test/java/com/palantir/witchcraft/java/logging/gradle/testreport/TestReportFormattingPluginIntegrationTest.java

@@ -1,5 +1,5 @@
 /*
- * (c) Copyright 2021 Palantir Technologies Inc. All rights reserved.
+ * (c) Copyright 2026 Palantir Technologies Inc. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,10 +25,33 @@
 import com.palantir.gradle.testing.junit.ParameterizedByGradleVersion;
 import com.palantir.gradle.testing.junit.ParameterizedByGradleVersion.WhenVersion;
 import com.palantir.gradle.testing.project.RootProject;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 @GradlePluginTests
 class TestReportFormattingPluginIntegrationTest {
+
+    private static final String LARGE_OUTPUT_TEST_CLASS = """
+        package com.palantir;
+        import org.junit.Test;
+
+        public final class LargeOutputTest {
+            private static final String SERVICE_JSON = "{\\"type\\":\\"service.1\\",\\"level\\":\\"ERROR\\","
+                + "\\"time\\":\\"2019-05-09T15:32:37.692Z\\",\\"origin\\":\\"ROOT\\","
+                + "\\"thread\\":\\"main\\",\\"message\\":\\"test good {}\\","
+                + "\\"params\\":{\\"good\\":\\":-)\\"},\\"unsafeParams\\":{},\\"tags\\":{}}";
+
+            @Test
+            public void largeOutputTest() {
+                // Print enough JSON log lines to create a report that will OOM the post-processor
+                for (int i = 0; i < 500_000; i++) {
+                    System.out.println(SERVICE_JSON);
+                }
+                throw new AssertionError("done");
+            }
+        }
+        """;
+
     private static final String SIMPLE_TEST_CLASS = """
         package com.palantir;
         import org.junit.Test;
@@ -66,15 +89,8 @@ public void simpleTest() {
         }
         """;
 
-    @Test
-    @ParameterizedByGradleVersion(
-            when =
-                    @WhenVersion(
-                            lessThan = "9.3.0",
-                            stringValue = "reports/tests/test/classes/com.palantir.SimpleTest.html"),
-            otherwiseString = "reports/tests/test/com.palantir.SimpleTest/simpleTest.html")
-    void formats_test_report_stdout_and_stderr(
-            GradleInvoker gradle, RootProject rootProject, @InjectByGradleVersion String reportPath) {
+    @BeforeEach
+    public void beforeEach(RootProject rootProject) {
         rootProject
                 .buildGradle()
                 .plugins()
@@ -102,7 +118,17 @@ void formats_test_report_stdout_and_stderr(
                 sourceCompatibility = 11
             }
             """);
+    }
 
+    @Test
+    @ParameterizedByGradleVersion(
+            when =
+                    @WhenVersion(
+                            lessThan = "9.3.0",
+                            stringValue = "reports/tests/test/classes/com.palantir.SimpleTest.html"),
+            otherwiseString = "reports/tests/test/com.palantir.SimpleTest/simpleTest.html")
+    void formats_test_report_stdout_and_stderr(
+            GradleInvoker gradle, RootProject rootProject, @InjectByGradleVersion String reportPath) {
         rootProject.testSourceSet().java().writeClass(SIMPLE_TEST_CLASS);
 
         gradle.withArgs("compileTestJava").buildsSuccessfully();
@@ -127,4 +153,32 @@ void formats_test_report_stdout_and_stderr(
                 .doesNotContain("Scavenge")
                 .contains("==Done==");
     }
+
+    @Test
+    @ParameterizedByGradleVersion(
+            when =
+                    @WhenVersion(
+                            lessThan = "9.3.0",
+                            stringValue = "reports/tests/test/classes/com.palantir.LargeOutputTest.html"),
+            otherwiseString = "reports/tests/test/com.palantir.LargeOutputTest/largeOutputTest.html")
+    void handles_large_report_without_oom(
+            GradleInvoker gradle, RootProject rootProject, @InjectByGradleVersion String reportPath) {
+        // Constrain Gradle daemon heap so in-memory processing will OOM
+        rootProject.gradlePropertiesFile().setProperty("org.gradle.jvmargs", "-Xmx256m");
+
+        rootProject.testSourceSet().java().writeClass(LARGE_OUTPUT_TEST_CLASS);
+
+        gradle.withArgs("compileTestJava").buildsSuccessfully();
+
+        gradle.withArgs("test").buildsWithFailure();
+
+        rootProject
+                .buildDir()
+                .file(reportPath)
+                .assertThat()
+                .content()
+                .contains("ERROR [2019-05-09T15:32:37.692Z]")
+                .doesNotContain("service.1")
+                .contains("done");
+    }
 }