Skip to content

Commit 014fc51

Browse files
Apache9Apache9
committed
HBASE-28679 Upgrade yetus to a newer version (#6012)
Signed-off-by: Nick Dimiduk <ndimiduk@apache.org> (cherry picked from commit 17ce7c3)
1 parent df16ceb commit 014fc51

File tree

5 files changed

+45
-43
lines changed

5 files changed

+45
-43
lines changed

‎dev-support/Jenkinsfile

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ pipeline {
3131
disableConcurrentBuilds()
3232
}
3333
environment {
34-
YETUS_RELEASE = '0.12.0'
34+
YETUS_RELEASE = '0.15.0'
3535
// where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
3636
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
3737
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
@@ -43,12 +43,12 @@ pipeline {
4343
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
4444
PERSONALITY_FILE = 'tools/personality.sh'
4545
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
46-
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
47-
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
46+
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
47+
BLANKS_IGNORE_LIST = '.*/generated/.*'
4848
// output from surefire; sadly the archive function in yetus only works on file names.
4949
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
5050
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
51-
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
51+
TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
5252
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes"
5353
// TODO does hadoopcheck need to be jdk specific?
5454
SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'

‎dev-support/Jenkinsfile_GitHub

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,16 @@ pipeline {
3636
YETUS_REL = 'yetus'
3737
DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
3838
YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
39-
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
40-
YETUS_VERSION = 'rel/0.12.0'
39+
YETUS_VERSION = '0.15.0'
4140
GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
4241
JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
42+
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
43+
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
44+
BLANKS_IGNORE_LIST = '.*/generated/.*'
4345
// output from surefire; sadly the archive function in yetus only works on file names.
4446
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
4547
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
46-
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
48+
TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
4749
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
4850
// set build parallel
4951
BUILD_THREAD = 4
@@ -102,11 +104,11 @@ pipeline {
102104
checkout scm
103105
}
104106
dir("${YETUSDIR}") {
105-
checkout([
106-
$class : 'GitSCM',
107-
branches : [[name: "${YETUS_VERSION}"]],
108-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
109-
)
107+
sh'''#!/usr/bin/env bash
108+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
109+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
110+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
111+
'''
110112
}
111113
dir("${WORKDIR}") {
112114
withCredentials([
@@ -229,11 +231,11 @@ pipeline {
229231
checkout scm
230232
}
231233
dir("${YETUSDIR}") {
232-
checkout([
233-
$class : 'GitSCM',
234-
branches : [[name: "${YETUS_VERSION}"]],
235-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
236-
)
234+
sh'''#!/usr/bin/env bash
235+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
236+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
237+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
238+
'''
237239
}
238240
dir("${WORKDIR}") {
239241
withCredentials([
@@ -365,11 +367,11 @@ pipeline {
365367
checkout scm
366368
}
367369
dir("${YETUSDIR}") {
368-
checkout([
369-
$class : 'GitSCM',
370-
branches : [[name: "${YETUS_VERSION}"]],
371-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
372-
)
370+
sh'''#!/usr/bin/env bash
371+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
372+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
373+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
374+
'''
373375
}
374376
dir("${WORKDIR}") {
375377
withCredentials([
@@ -493,22 +495,19 @@ pipeline {
493495
SKIP_ERRORPRONE = true
494496
}
495497
when {
496-
allOf {
497-
// this will return true if the pipeline is building a change request, such as a GitHub pull request.
498-
changeRequest()
499-
expression { env.CHANGE_TARGET in ['master', 'branch-3'] }
500-
}
498+
// this will return true if the pipeline is building a change request, such as a GitHub pull request.
499+
changeRequest()
501500
}
502501
steps {
503502
dir("${SOURCEDIR}") {
504503
checkout scm
505504
}
506505
dir("${YETUSDIR}") {
507-
checkout([
508-
$class : 'GitSCM',
509-
branches : [[name: "${YETUS_VERSION}"]],
510-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
511-
)
506+
sh'''#!/usr/bin/env bash
507+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
508+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
509+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
510+
'''
512511
}
513512
dir("${WORKDIR}") {
514513
withCredentials([

‎dev-support/docker/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
5353
ruby=1:3.0* \
5454
ruby-dev=1:3.0* \
5555
shellcheck='0.8.0-*' \
56+
libxml2-dev='2.9.13+dfsg-*' \
57+
libxml2-utils='2.9.13+dfsg-*' \
5658
&& \
5759
apt-get clean && \
5860
rm -rf /var/lib/apt/lists/* \

‎dev-support/hbase_nightly_yetus.sh

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ declare -i missing_env=0
2020
# Validate params
2121
for required_env in "TESTS" "PERSONALITY_FILE" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE" \
2222
"OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST" \
23-
"WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
23+
"BLANKS_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
2424
"USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do
2525
if [ -z "${!required_env}" ]; then
2626
echo "[ERROR] Required environment variable '${required_env}' is not set."
@@ -59,8 +59,8 @@ YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}")
5959
YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}")
6060
YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}")
6161
YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}")
62-
YETUS_ARGS=("--whitespace-eol-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
63-
YETUS_ARGS=("--whitespace-tabs-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
62+
YETUS_ARGS=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
63+
YETUS_ARGS=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
6464
YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
6565
YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
6666
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
@@ -106,11 +106,9 @@ if [[ -n "${JAVA8_HOME}" ]]; then
106106
fi
107107

108108
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
109-
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
110109
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
111110
else
112-
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-git/shelldocs/shelldocs.py" "${YETUS_ARGS[@]}")
113-
TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/test-patch.sh"
111+
TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/src/main/shell/test-patch.sh"
114112
fi
115113
echo "Launching yetus with command line:"
116114
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"

‎dev-support/jenkins_precommit_github_yetus.sh

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ declare -a required_envs=(
4141
"SOURCEDIR"
4242
"TESTS_FILTER"
4343
"YETUSDIR"
44+
"AUTHOR_IGNORE_LIST"
45+
"BLANKS_IGNORE_LIST"
4446
)
4547
# Validate params
4648
for required_env in "${required_envs[@]}"; do
@@ -57,7 +59,7 @@ if [ ${missing_env} -gt 0 ]; then
5759
fi
5860

5961
# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
60-
TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
62+
TESTPATCHBIN="${YETUSDIR}/bin/test-patch"
6163

6264
# this must be clean for every run
6365
rm -rf "${PATCHDIR}"
@@ -87,8 +89,8 @@ YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
8789
YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
8890
YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
8991
# enable writing back to Github
90-
YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
91-
YETUS_ARGS+=("--github-user=${GITHUB_USER}")
92+
YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}")
93+
YETUS_ARGS+=("--github-write-comment")
9294
# auto-kill any surefire stragglers during unit test runs
9395
YETUS_ARGS+=("--reapermode=kill")
9496
# set relatively high limits for ASF machines
@@ -109,8 +111,9 @@ YETUS_ARGS+=("--docker")
109111
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
110112
YETUS_ARGS+=("--mvn-custom-repos")
111113
YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
112-
YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
113-
YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
114+
YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}")
115+
YETUS_ARGS+=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}")
116+
YETUS_ARGS+=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}*")
114117
YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
115118
YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
116119
YETUS_ARGS+=("--quick-hadoopcheck")

0 commit comments

Comments
 (0)