HDDS-2030. Generate simplifed reports by the dev-support/checks/*.sh scripts

hadoop-ozone/dev-support/checks/README.md

@@ -0,0 +1,27 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Ozone checks
+
+This directory contains a collection of easy-to-use helper scripts to execute various type of tests on the ozone/hdds codebase.
+
+The contract of the scripts are very simple:
+
+ 1. Executing the scripts without any parameter will check the hdds/ozone project
+ 2. Shell exit code represents the result of the check (if failed, exits with non-zero code)
+ 3. Detailed information may be saved to the $OUTPUT_DIR (if it's not set, root level ./target will be used).
+ 4. The standard output should contain all the log about the build AND the results.
+ 5. The content of the $OUTPUT_DIR can be:
+    * `summary.html`/`summary.md`/`summary.txt`: contains a human readable overview about the failed tests (used by reporting)
+    * `failures`: contains a simple number (used by reporting)

hadoop-ozone/dev-support/checks/_mvn_unit_report.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPORT_DIR=${REPORT_DIR:-$PWD}
+
+## generate summary txt file
+find "." -name 'TEST*.xml' -print0 \
+    | xargs -n1 -0 "grep" -l -E "<failure|<error" \
+    | awk -F/ '{sub("'"TEST-"'",""); sub(".xml",""); print $NF}' \
+    | tee "$REPORT_DIR/summary.txt"
+
+#Copy heap dump and dump leftovers
+find "." -name "*.hprof" -exec cp {} "$REPORT_DIR/" \;
+find "." -name "*.dump" -exec cp {} "$REPORT_DIR/" \;
+
+## Add the tests where the JVM is crashed
+grep -A1 'Crashed tests' "${REPORT_DIR}/output.log" \
+  | grep -v -e 'Crashed tests' -e '--' \
+  | cut -f2- -d' ' \
+  | sort -u >> "${REPORT_DIR}/summary.txt"
+
+#Collect of all of the report failes of FAILED tests
+while IFS= read -r -d '' dir; do
+   while IFS=$'\n' read -r file; do
+      DIR_OF_TESTFILE=$(dirname "$file")
+      NAME_OF_TESTFILE=$(basename "$file")
+      NAME_OF_TEST="${NAME_OF_TESTFILE%.*}"
+      DESTDIRNAME=$(realpath --relative-to="$PWD" "$DIR_OF_TESTFILE/../..")
+      mkdir -p "$REPORT_DIR/$DESTDIRNAME"
+      #shellcheck disable=SC2086
+      cp -r "$DIR_OF_TESTFILE"/*$NAME_OF_TEST* "$REPORT_DIR/$DESTDIRNAME/"
+   done < <(grep -l -r FAILURE --include="*.txt" "$dir" | grep -v output.txt)
+done < <(find "." -name surefire-reports -print0)
+
+## generate summary markdown file
+export SUMMARY_FILE="$REPORT_DIR/summary.md"
+for TEST_RESULT_FILE in $(find "$REPORT_DIR" -name "*.txt" | grep -v output); do
+
+    FAILURES=$(grep FAILURE "$TEST_RESULT_FILE" | grep "Tests run" | awk '{print $18}' | sort | uniq)
+
+    for FAILURE in $FAILURES; do
+        TEST_RESULT_LOCATION="$(realpath --relative-to="$REPORT_DIR" "$TEST_RESULT_FILE")"
+        TEST_OUTPUT_LOCATION="${TEST_RESULT_LOCATION//.txt/-output.txt/}"
+        printf " * [%s](%s) ([output](%s))\n" "$FAILURE" "$TEST_RESULT_LOCATION" "$TEST_OUTPUT_LOCATION" >> "$SUMMARY_FILE"
+    done
+done
+
+if [ -s "$SUMMARY_FILE" ]; then
+   printf "# Failing tests: \n\n" | cat - "$SUMMARY_FILE" > temp && mv temp "$SUMMARY_FILE"
+fi
+
+## generate counter
+wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"

hadoop-ozone/dev-support/checks/acceptance.sh

@@ -16,7 +16,20 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/acceptance"}
+mkdir -p "$REPORT_DIR"
+
 OZONE_VERSION=$(grep "<ozone.version>" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'|  sed 's/^[ \t]*//')
-cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1
+DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
+
+if [ ! -d "$DIST_DIR" ]; then
+    echo "Distribution dir is missing. Doing a full build"
+    "$DIR/build.sh"
+fi
+
+cd "$DIST_DIR/compose" || exit 1
 ./test-all.sh
-exit $?
+RES=$?
+cp result/* "$REPORT_DIR/"
+cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
+exit $RES

hadoop-ozone/dev-support/checks/author.sh

@@ -16,12 +16,18 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
-#hide this tring to not confuse yetus
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/author"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
+#hide this string to not confuse yetus
 AUTHOR="uthor"
 AUTHOR="@a${AUTHOR}"
 
-if grep -r --include="*.java" "$AUTHOR" .; then
-  exit 1
-else
-  exit 0
+grep -r --include="*.java" "$AUTHOR" . | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
+
+if [[ -s "${REPORT_FILE}" ]]; then
+   exit 1
 fi

hadoop-ozone/dev-support/checks/checkstyle.sh

@@ -16,14 +16,18 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/checkstyle"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
 mvn -B -fn checkstyle:check -f pom.ozone.xml
 
 #Print out the exact violations with parsing XML results with sed
-find "." -name checkstyle-errors.xml -print0  | xargs -0 sed  '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
+find "." -name checkstyle-errors.xml -print0  | xargs -0 sed  '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d' | tee "$REPORT_FILE"
+
+## generate counter
+wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"
 
-violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
-if [[ $violations -gt 0 ]]; then
-    echo "There are $violations checkstyle violations"
-    exit 1
+if [[ -s "${REPORT_FILE}" ]]; then
+   exit 1
 fi
-exit 0

hadoop-ozone/dev-support/checks/findbugs.sh

@@ -16,21 +16,19 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
-FINDBUGS_ALL_FILE=./target/findbugs-all.txt
+mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false  -f pom.ozone.xml
 
-mkdir -p ./target
-rm "$FINDBUGS_ALL_FILE" || true
-touch "$FINDBUGS_ALL_FILE"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
 
-mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false  -f pom.ozone.xml
+touch "$REPORT_FILE"
 
-find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-find hadoop-hdds -name findbugsXml.xml -print0  | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}"
+find hadoop-hdds -name findbugsXml.xml -print0  | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}"
 
-bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
 
-if [[ ${bugs} -gt 0 ]]; then
+if [[ -s "${REPORT_FILE}" ]]; then
    exit 1
-else
-   exit 0
 fi

hadoop-ozone/dev-support/checks/integration.sh

@@ -20,10 +20,14 @@ export MAVEN_OPTS="-Xmx4096m"
 mvn -B install -f pom.ozone.xml -DskipTests
 mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem,:hadoop-ozone-tools \
   -Dtest=\!TestMiniChaosOzoneCluster
-module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
-    | xargs -0 -n1 "grep" -l -E "<failure|<error"\
-    | awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
-if [[ -n "${module_failed_tests}" ]] ; then
+
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/integration"}
+mkdir -p "$REPORT_DIR"
+
+# shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
+source "$DIR/_mvn_unit_report.sh"
+
+if [[ -s "$REPORT_DIR/summary.txt" ]] ; then
     exit 1
 fi
 exit 0

hadoop-ozone/dev-support/checks/rat.sh

@@ -16,17 +16,22 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
-mkdir -p target
-REPORT_FILE="$DIR/../../../target/rat-aggregated.txt"
-mkdir -p "$(dirname "$REPORT_FILE")"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/rat"}
+mkdir -p "$REPORT_DIR"
+
+REPORT_FILE="$REPORT_DIR/summary.txt"
 
 cd hadoop-hdds || exit 1
 mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
 cd ../hadoop-ozone || exit 1
 mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
 
 cd "$DIR/../../.." || exit 1
+
 grep -r --include=rat.txt "!????" hadoop-hdds hadoop-ozone | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
+
 if [[ -s "${REPORT_FILE}" ]]; then
    exit 1
 fi

hadoop-ozone/dev-support/checks/shellcheck.sh

@@ -16,8 +16,10 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
-OUTPUT_FILE="$DIR/../../../target/shell-problems.txt"
-mkdir -p "$(dirname "$OUTPUT_FILE")"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/shellcheck"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
 echo "" > "$OUTPUT_FILE"
 if [[ "$(uname -s)" = "Darwin" ]]; then
   find hadoop-hdds hadoop-ozone -type f -perm '-500'
@@ -26,8 +28,10 @@ else
 fi \
   | grep -v -e target/ -e node_modules/ -e '\.\(ico\|py\|yml\)$' \
   | xargs -n1 shellcheck \
-  | tee "$OUTPUT_FILE"
+  | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
 
-if [ "$(cat "$OUTPUT_FILE")" ]; then
+if [[ -s "${REPORT_FILE}" ]]; then
    exit 1
 fi

hadoop-ozone/dev-support/checks/unit.sh

@@ -13,12 +13,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
 export MAVEN_OPTS="-Xmx4096m"
-mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools
-module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
-    | xargs -n1 -0 "grep" -l -E "<failure|<error"\
-    | awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
-if [[ -n "${module_failed_tests}" ]] ; then
+mvn -B -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools
+
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/unit"}
+mkdir -p "$REPORT_DIR"
+
+# shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
+source "$DIR/_mvn_unit_report.sh"
+
+if [[ -s "$REPORT_DIR/summary.txt" ]] ; then
     exit 1
 fi
 exit 0