Skip to content

HDDS-16935. Distinguish CI checks and splits in Develocity #11492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
adoroszlai wants to merge 1 commit into
base: master
Choose a base branch
from
+43 −12
Draft

HDDS-16935. Distinguish CI checks and splits in Develocity #11492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions hadoop-ozone/dev-support/checks/_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
REPORT_FILE="$REPORT_DIR/summary.txt"

MAVEN_OPTIONS='-V -B -DskipTests -DskipDocs --no-transfer-progress'
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"

if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Pcoverage"
Expand Down
5 changes: 4 additions & 1 deletion hadoop-ozone/dev-support/checks/checkstyle.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,15 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

CHECK=checkstyle

BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/checkstyle"}
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
mkdir -p "$REPORT_DIR"
REPORT_FILE="$REPORT_DIR/summary.txt"

MAVEN_OPTIONS='-B -fae -DskipDocs -DskipRecon -Dcheckstyle.failOnViolation=false --no-transfer-progress'
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"

declare -i rc
mvn ${MAVEN_OPTIONS} checkstyle:check > "${REPORT_DIR}/output.log"
Expand Down
6 changes: 5 additions & 1 deletion hadoop-ozone/dev-support/checks/errorprone.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,16 @@ set -u -o pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/errorprone"}
CHECK=errorprone

BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
REPORT_FILE="$REPORT_DIR/summary.txt"
DIAGNOSTIC_FILE="$REPORT_DIR/diagnostics.txt"
OUTPUT_LOG=$(mktemp)

MAVEN_OPTIONS='-B -fae --no-transfer-progress -Perrorprone -DskipDocs -DskipRecon -DskipShade -DskipTests'
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"
MAVEN_DIAGNOSTIC_PATTERN='^\[(ERROR|WARNING)\] .*:\[[0-9]+,[0-9]+\] \[[^]]+\]'
JAVAC_DIAGNOSTIC_PATTERN='^.*:[0-9]+: (error|warning): \[[^]]+\]'
MAVEN_ERROR_PATTERN='^\[ERROR\] .*:\[[0-9]+,[0-9]+\] \[[^]]+\]'
Expand Down
6 changes: 5 additions & 1 deletion hadoop-ozone/dev-support/checks/findbugs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,18 @@ cd "$DIR/../../.." || exit 1

: ${OZONE_WITH_COVERAGE:="false"}

REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"}
CHECK=findbugs

BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
mkdir -p "$REPORT_DIR"
REPORT_FILE="$REPORT_DIR/summary.txt"

source "${DIR}/_lib.sh"
source "${DIR}/install/spotbugs.sh"

MAVEN_OPTIONS='-B -fae -DskipDocs -DskipRecon -DskipShade --no-transfer-progress'
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"

if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
Expand Down
9 changes: 6 additions & 3 deletions hadoop-ozone/dev-support/checks/integration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,15 @@ PHASE=test

args=""

if [[ "$@" =~ "-Ptest-" ]]; then
if [[ "$@" =~ "-Ptest-flaky" ]]; then
if [[ "$@" =~ -Ptest-([a-z]+) ]]; then
split="${BASH_REMATCH[1]}"
if [[ "${split}" == "flaky" ]]; then
args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
# tests may pass on re-run, so we cannot rely on output for status
ERROR_PATTERN=""
fi

if [[ "$@" =~ "-Ptest-filesystem" ]]; then
if [[ "${split}" == "filesystem" ]]; then
if [[ "${CI:-}" == "true" ]]; then
args="$args -DskipShade --activate-profiles CI"
else
Expand All @@ -38,6 +39,8 @@ if [[ "$@" =~ "-Ptest-" ]]; then
else
args="$args -DskipShade"
fi

args="$args -Dscan.value.testset=${split}"
fi

source "${DIR}/junit.sh" $args "$@"
5 changes: 4 additions & 1 deletion hadoop-ozone/dev-support/checks/javadoc.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,14 @@ set -u -o pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

CHECK=javadoc

BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/javadoc"}
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
REPORT_FILE="$REPORT_DIR/summary.txt"

MAVEN_OPTIONS="-B -fae -DskipDocs -DskipRecon -DskipShade --no-transfer-progress ${MAVEN_OPTIONS:-}"
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"

mvn ${MAVEN_OPTIONS} javadoc:aggregate "$@" | tee output.log
rc=$?
Expand Down
6 changes: 4 additions & 2 deletions hadoop-ozone/dev-support/checks/junit.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ fi

export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}"
MAVEN_OPTIONS="-B -V -DskipDocs -DskipRecon -Dnative.lib.tmp.dir=/tmp --no-transfer-progress"
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"

if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
Expand All @@ -55,13 +56,14 @@ if [[ -f hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh ]]; then
ozone_java_setup
fi

mvn ${MAVEN_OPTIONS} clean
mvn ${MAVEN_OPTIONS} clean -Dscan=false

if [[ ${ITERATIONS} -gt 1 ]] && [[ ${OZONE_REPO_CACHED} == "false" ]]; then
mvn ${MAVEN_OPTIONS} -DskipTests install
fi

REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/${CHECK}"}
BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
REPORT_FILE="${REPORT_DIR}/summary.txt"
mkdir -p "$REPORT_DIR"

Expand Down
5 changes: 4 additions & 1 deletion hadoop-ozone/dev-support/checks/license.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ set -euo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/license"}
CHECK=license

BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
mkdir -p "$REPORT_DIR"
REPORT_FILE="${REPORT_DIR}/summary.txt"

Expand Down
6 changes: 5 additions & 1 deletion hadoop-ozone/dev-support/checks/pmd.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,16 @@ set -u -o pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/pmd"}
CHECK=pmd

BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
mkdir -p "$REPORT_DIR"

REPORT_FILE="$REPORT_DIR/summary.txt"

MAVEN_OPTIONS='-B -fae --no-transfer-progress -Dpmd.failOnViolation=false -Dpmd.printFailingErrors -DskipDocs -DskipRecon -DskipShade'
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"

declare -i rc

Expand Down
5 changes: 4 additions & 1 deletion hadoop-ozone/dev-support/checks/rat.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/rat"}
CHECK=rat

BASE_DIR="$(pwd -P)"
REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/${CHECK}"}
mkdir -p "$REPORT_DIR"

REPORT_FILE="$REPORT_DIR/summary.txt"
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ EOF
# - ITestS3AContractRename: HDDS-10665
mvn ${MAVEN_ARGS:-} --fail-never --show-version \
-Dtest='ITestS3AContract*, ITestS3ACommitterMRJob, !ITestS3AContractBulkDelete, !ITestS3AContractCreate#testOverwrite*EmptyDirectory[*], !ITestS3AContractDistCp, !ITestS3AContractMkdirWithCreatePerf, !ITestS3AContractRename' \
-Dscan.value.testset=s3a \
clean test

local target="${RESULT_DIR}/junit/${bucket}/target"
Expand Down