Show errors in Markdown and do not fail the grading

.github/workflows/autograding.yml

@@ -0,0 +1,124 @@
+name: 'Autograding PR'
+
+on:
+  pull_request:
+
+jobs:
+  build:
+
+    runs-on: [ubuntu-latest]
+    name: Build, test and autograde on Ubuntu
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+          check-latest: true
+          cache: 'maven'
+      - name: Set up Maven
+        uses: stCarolas/setup-maven@v4.5
+        with:
+          maven-version: 3.9.5
+      - name: Build with Maven
+        env:
+          BROWSER: chrome-container
+        run: mvn -V --color always -ntp clean verify --file pom.xml '-Djenkins.test.timeout=5000' '-Dgpg.skip' -Ppit | tee maven.log
+      - name: Extract pull request number
+        uses: jwalton/gh-find-current-pr@v1
+        id: pr
+      - name: Run Autograding
+        uses: uhafner/autograding-github-action@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          pr-number: ${{ steps.pr.outputs.number }}
+          checks-name: 'Quality Checks'
+          config: >
+            {
+              "tests": {
+                "tools": [
+                  {
+                    "id": "test",
+                    "name": "Unittests",
+                    "pattern": "**/target/*-reports/TEST*.xml"
+                  }
+                ],
+                "name": "JUnit",
+                "passedImpact": 0,
+                "skippedImpact": -1,
+                "failureImpact": -5,
+                "maxScore": 100
+              },
+              "analysis": {
+                "name": "Warnings",
+                "id": "warnings",
+                "tools": [
+                  {
+                    "id": "checkstyle",
+                    "name": "CheckStyle",
+                    "pattern": "**/target/checkstyle-result.xml"
+                  },
+                  {
+                    "id": "pmd",
+                    "name": "PMD",
+                    "pattern": "**/target/pmd.xml"
+                  },
+                  {
+                    "id": "error-prone",
+                    "name": "Error Prone",
+                    "pattern": "**/maven.log"
+                  },
+                  {
+                    "id": "spotbugs",
+                    "name": "SpotBugs",
+                    "sourcePath": "src/main/java",
+                    "pattern": "**/target/spotbugsXml.xml"
+                  }
+
+                ],
+                "errorImpact": -1,
+                "highImpact": -1,
+                "normalImpact": -1,
+                "lowImpact": -1,
+                "maxScore": 100
+              },
+              "coverage": [
+                {
+                  "tools": [
+                    {
+                      "id": "jacoco",
+                      "name": "Line Coverage",
+                      "metric": "line",
+                      "sourcePath": "src/main/java",
+                      "pattern": "**/target/site/jacoco/jacoco.xml"
+                    },
+                    {
+                      "id": "jacoco",
+                      "name": "Branch Coverage",
+                      "metric": "branch",
+                      "sourcePath": "src/main/java",
+                      "pattern": "**/target/site/jacoco/jacoco.xml"
+                    }
+                  ],
+                  "name": "JaCoCo",
+                  "maxScore": 100,
+                  "missedPercentageImpact": -1
+                },
+                {
+                  "tools": [
+                    {
+                      "id": "pit",
+                      "name": "Mutation Coverage",
+                      "metric": "mutation",
+                      "sourcePath": "src/main/java",
+                      "pattern": "**/target/pit-reports/mutations.xml"
+                    }
+                  ],
+                  "name": "PIT",
+                  "maxScore": 100,
+                  "missedPercentageImpact": -1
+                }
+              ]
+            }

doc/dependency-graph.puml

@@ -97,4 +97,4 @@ edu_hm_hafner_autograding_github_action_jar -[#000000]-> com_github_spotbugs_spo
 edu_hm_hafner_autograding_github_action_jar -[#000000]-> com_google_errorprone_error_prone_annotations_jar
 edu_hm_hafner_autograding_github_action_jar -[#000000]-> one_util_streamex_jar
 edu_hm_hafner_autograding_github_action_jar -[#000000]-> edu_hm_hafner_codingstyle_jar
-@enduml
+@enduml

src/main/java/edu/hm/hafner/grading/AutoGradingAction.java

@@ -6,6 +6,7 @@
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.NoSuchElementException;
+import java.util.StringJoiner;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -74,10 +75,12 @@ void run() {
 
             System.out.println("Commenting on commit or pull request...");
 
+            var errors = createErrorMessage(log);
+
             pullRequestWriter.addComment(getChecksName(), score,
                     results.getHeader(), results.getTextSummary(score),
-                    results.getMarkdownDetails(score),
-                    results.getMarkdownSummary(score, ":mortar_board: " + getChecksName()),
+                    results.getMarkdownDetails(score) + errors,
+                    results.getMarkdownSummary(score, ":mortar_board: " + getChecksName()) + errors,
                     ChecksStatus.SUCCESS);
 
             var environmentVariables = createEnvironmentVariables(score);
@@ -104,6 +107,21 @@ void run() {
         System.out.println("------------------------------------------------------------------");
     }
 
+    private String createErrorMessage(final FilteredLog log) {
+        if (log.hasErrors()) {
+            StringBuilder errors = new StringBuilder();
+
+            errors.append("## Error Messages\n```\n");
+            var messages = new StringJoiner("\n");
+            log.getErrorMessages().forEach(messages::add);
+            errors.append(messages);
+            errors.append("\n```\n");
+
+            return errors.toString();
+        }
+        return StringUtils.EMPTY;
+    }
+
     String createEnvironmentVariables(final AggregatedScore score) {
         var metrics = new StringBuilder();
         score.getMetrics().forEach((metric, value) -> metrics.append(String.format("%s=%d%n", metric, value)));

src/main/java/edu/hm/hafner/grading/ConsoleAnalysisReportFactory.java

@@ -14,15 +14,17 @@
  *
  * @author Ullrich Hafner
  */
-public final class ConsoleAnalysisReportFactory extends ReportFactory implements AnalysisReportFactory {
+public final class ConsoleAnalysisReportFactory implements AnalysisReportFactory {
+    private static final ReportFinder REPORT_FINDER = new ReportFinder();
+
     @Override
     public Report create(final ToolConfiguration tool, final FilteredLog log) {
         ParserDescriptor parser = new ParserRegistry().get(tool.getId());
 
         var total = new Report(tool.getId(), tool.getDisplayName());
 
         var analysisParser = parser.createParser();
-        for (Path file : findFiles(tool, log)) {
+        for (Path file : REPORT_FINDER.find(tool, log)) {
             Report report = analysisParser.parse(new FileReaderFactory(file));
             report.setOrigin(tool.getId(), tool.getDisplayName());
             log.logInfo("- %s: %d warnings", file, report.size());

src/main/java/edu/hm/hafner/grading/ConsoleCoverageReportFactory.java

@@ -20,27 +20,38 @@
  *
  * @author Ullrich Hafner
  */
-public final class ConsoleCoverageReportFactory extends ReportFactory implements CoverageReportFactory {
+public final class ConsoleCoverageReportFactory implements CoverageReportFactory {
+    private static final ReportFinder REPORT_FINDER = new ReportFinder();
+
     @Override
     public Node create(final ToolConfiguration tool, final FilteredLog log) {
         var parser = new ParserRegistry().getParser(StringUtils.upperCase(tool.getId()), ProcessingMode.FAIL_FAST);
 
         var nodes = new ArrayList<Node>();
-        for (Path file : findFiles(tool, log)) {
+        for (Path file : REPORT_FINDER.find(tool, log)) {
             var node = parser.parse(new FileReaderFactory(file).create(), log);
             log.logInfo("- %s: %s", file, extractMetric(tool, node));
             nodes.add(node);
         }
 
-        var aggregation = Node.merge(nodes);
-        log.logInfo("-> %s Total: %s", tool.getDisplayName(), extractMetric(tool, aggregation));
-        if (tool.getName().isBlank()) {
-            return aggregation;
+        if (nodes.isEmpty()) {
+            return createContainer(tool);
+        }
+        else {
+            var aggregation = Node.merge(nodes);
+            log.logInfo("-> %s Total: %s", tool.getDisplayName(), extractMetric(tool, aggregation));
+            if (tool.getName().isBlank()) {
+                return aggregation;
+            }
+            // Wrap the node into a container with the specified tool name
+            var containerNode = createContainer(tool);
+            containerNode.addChild(aggregation);
+            return containerNode;
         }
-        // Wrap the node into a container with the specified tool name
-        var containerNode = new ContainerNode(tool.getName());
-        containerNode.addChild(aggregation);
-        return containerNode;
+    }
+
+    private ContainerNode createContainer(final ToolConfiguration tool) {
+        return new ContainerNode(tool.getName());
     }
 
     private String extractMetric(final ToolConfiguration tool, final Node node) {

src/main/java/edu/hm/hafner/grading/LogHandler.java

@@ -58,7 +58,7 @@ private void printInfoMessages() {
     private void printErrorMessages() {
         var size = getSizeOfErrorMessages();
         if (errorPosition < size && !quiet) {
-            logger.getErrorMessages().subList(infoPosition, size).forEach(printStream::println);
+            logger.getErrorMessages().subList(errorPosition, size).forEach(printStream::println);
             errorPosition = logger.getErrorMessages().size();
         }
     }

src/main/java/edu/hm/hafner/grading/ReportFinder.java

@@ -11,11 +11,13 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
+import edu.hm.hafner.util.FilteredLog;
 import edu.hm.hafner.util.VisibleForTesting;
 
 /**
@@ -42,6 +44,30 @@ private static String getEnv(final String name) {
         this.branch = branch;
     }
 
+    /**
+     * Returns the paths for the specified tool.
+     *
+     * @param tool
+     *         the tool to find the reports for
+     * @param log
+     *         logger
+     *
+     * @return the paths
+     */
+    public List<Path> find(final ToolConfiguration tool, final FilteredLog log) {
+        log.logInfo("Searching for %s results matching file name pattern %s",
+                tool.getDisplayName(), tool.getPattern());
+        List<Path> files = new ReportFinder().find("glob:" + tool.getPattern());
+
+        if (files.isEmpty()) {
+            log.logError("No matching report files found when using pattern '%s'! "
+                    + "Configuration error for '%s'?", tool.getPattern(), tool.getDisplayName());
+        }
+
+        Collections.sort(files);
+        return files;
+    }
+
     /**
      * Returns the paths that match the specified pattern.
      *
@@ -100,7 +126,8 @@ private static class PathMatcherFileVisitor extends SimpleFileVisitor<Path> {
                 pathMatcher = FileSystems.getDefault().getPathMatcher(syntaxAndPattern);
             }
             catch (IllegalArgumentException exception) {
-                throw new IllegalArgumentException("Pattern not valid for FileSystem.getPathMatcher: " + syntaxAndPattern, exception);
+                throw new IllegalArgumentException(
+                        "Pattern not valid for FileSystem.getPathMatcher: " + syntaxAndPattern, exception);
             }
         }
 

src/main/java/edu/hm/hafner/grading/github/GitHubPullRequestWriter.java

@@ -15,6 +15,7 @@
 import edu.hm.hafner.analysis.Issue;
 import edu.hm.hafner.analysis.Report;
 import edu.hm.hafner.coverage.FileNode;
+import edu.hm.hafner.coverage.Metric;
 import edu.hm.hafner.coverage.Mutation;
 import edu.hm.hafner.coverage.Node;
 import edu.hm.hafner.grading.AggregatedScore;
@@ -195,6 +196,7 @@ private String cleanFileName(final String prefix, final String fileName, final S
     private void createLineAnnotationsForMissedLines(final AggregatedScore score, final String prefix,
             final Set<String> prefixes, final Output output) {
         score.getCodeCoverageScores().stream()
+                .filter(coverageScore -> coverageScore.getMetric() == Metric.LINE)
                 .map(CoverageScore::getReport)
                 .map(Node::getAllFileNodes)
                 .flatMap(Collection::stream)
@@ -231,6 +233,7 @@ private String getMissedLinesDescription(final LineRange range) {
     private void createLineAnnotationsForPartiallyCoveredLines(final AggregatedScore score, final String prefix,
             final Set<String> prefixes, final Output output) {
         score.getCodeCoverageScores().stream()
+                .filter(coverageScore -> coverageScore.getMetric() == Metric.BRANCH)
                 .map(CoverageScore::getReport)
                 .map(Node::getAllFileNodes)
                 .flatMap(Collection::stream)
@@ -261,6 +264,7 @@ private String createBranchMessage(final int line, final int missed) {
     private void createLineAnnotationsForSurvivedMutations(final AggregatedScore score, final String prefix,
             final Set<String> prefixes, final Output output) {
         score.getMutationCoverageScores().stream()
+                .filter(coverageScore -> coverageScore.getMetric() == Metric.MUTATION)
                 .map(CoverageScore::getReport)
                 .map(Node::getAllFileNodes)
                 .flatMap(Collection::stream)