Skip to content

Commit 3722dc9

Browse files
Apache9Apache9
committed
HBASE-28677 Add jdk 17 task for pre commit build for 2.x (#6002)
Signed-off-by: Nick Dimiduk <ndimiduk@apache.org> (cherry picked from commit 8fd1eca) (cherry picked from commit 3001306)
1 parent 47b1bb3 commit 3722dc9

File tree

2 files changed

+153
-0
lines changed

2 files changed

+153
-0
lines changed

‎dev-support/Jenkinsfile_GitHub

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ pipeline {
5858
WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
5959
WORKDIR_REL_JDK8_HADOOP2_CHECK = 'yetus-jdk8-hadoop2-check'
6060
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
61+
WORKDIR_REL_JDK17_HADOOP3_CHECK = 'yetus-jdk17-hadoop3-check'
6162
ASF_NIGHTLIES = 'https://nightlies.apache.org'
6263
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
6364
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
@@ -474,6 +475,143 @@ pipeline {
474475
}
475476
}
476477
}
478+
stage ('yetus jdk17 hadoop3 checks') {
479+
agent {
480+
node {
481+
label 'hbase'
482+
}
483+
}
484+
environment {
485+
// customized per parallel stage
486+
PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
487+
SET_JAVA_HOME = '/usr/lib/jvm/java-17'
488+
HADOOP_PROFILE = '3.0'
489+
WORKDIR_REL = "${WORKDIR_REL_JDK17_HADOOP3_CHECK}"
490+
// identical for all parallel stages
491+
WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
492+
YETUSDIR = "${WORKDIR}/${YETUS_REL}"
493+
SOURCEDIR = "${WORKDIR}/${SRC_REL}"
494+
PATCHDIR = "${WORKDIR}/${PATCH_REL}"
495+
BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
496+
DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
497+
YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
498+
SKIP_ERRORPRONE = true
499+
}
500+
when {
501+
// this will return true if the pipeline is building a change request, such as a GitHub pull request.
502+
changeRequest()
503+
}
504+
steps {
505+
dir("${SOURCEDIR}") {
506+
checkout scm
507+
}
508+
dir("${YETUSDIR}") {
509+
sh'''#!/usr/bin/env bash
510+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
511+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
512+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
513+
'''
514+
}
515+
dir("${WORKDIR}") {
516+
withCredentials([
517+
usernamePassword(
518+
credentialsId: 'apache-hbase-at-github.com',
519+
passwordVariable: 'GITHUB_PASSWORD',
520+
usernameVariable: 'GITHUB_USER'
521+
)]) {
522+
script {
523+
def ret = sh(
524+
label: 'test-patch',
525+
returnStatus: true,
526+
script: '''#!/bin/bash -e
527+
hostname -a ; pwd ; ls -la
528+
printenv 2>&1 | sort
529+
echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
530+
"${YETUS_DRIVER}"
531+
'''
532+
)
533+
if (ret != 0) {
534+
// mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
535+
// test output. See HBASE-26339 for more details.
536+
currentBuild.result = 'UNSTABLE'
537+
}
538+
}
539+
}
540+
}
541+
}
542+
post {
543+
always {
544+
junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
545+
allowEmptyResults: true, skipPublishingChecks: true
546+
sh label: 'zip surefire reports', script: '''#!/bin/bash -e
547+
if [ -d "${PATCHDIR}/archiver" ]; then
548+
count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
549+
if [[ 0 -ne ${count} ]]; then
550+
echo "zipping ${count} archived files"
551+
zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
552+
else
553+
echo "No archived files, skipping compressing."
554+
fi
555+
else
556+
echo "No archiver directory, skipping compressing."
557+
fi
558+
'''
559+
sshPublisher(publishers: [
560+
sshPublisherDesc(configName: 'Nightlies',
561+
transfers: [
562+
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
563+
sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
564+
)
565+
]
566+
)
567+
])
568+
// remove the big test logs zip file, store the nightlies url in test_logs.txt
569+
sh '''#!/bin/bash -e
570+
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
571+
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
572+
rm -rf "${PATCHDIR}/test_logs.zip"
573+
python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
574+
else
575+
echo "No test_logs.zip, skipping"
576+
fi
577+
'''
578+
// Has to be relative to WORKSPACE.
579+
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
580+
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
581+
publishHTML target: [
582+
allowMissing: true,
583+
keepAll: true,
584+
alwaysLinkToLastBuild: true,
585+
// Has to be relative to WORKSPACE
586+
reportDir: "${WORKDIR_REL}/${PATCH_REL}",
587+
reportFiles: 'report.html',
588+
reportName: 'PR JDK17 Hadoop3 Check Report'
589+
]
590+
}
591+
// Jenkins pipeline jobs fill slaves on PRs without this :(
592+
cleanup() {
593+
script {
594+
sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
595+
# See YETUS-764
596+
if [ -f "${PATCHDIR}/pidfile.txt" ]; then
597+
echo "test-patch process appears to still be running: killing"
598+
kill `cat "${PATCHDIR}/pidfile.txt"` || true
599+
sleep 10
600+
fi
601+
if [ -f "${PATCHDIR}/cidfile.txt" ]; then
602+
echo "test-patch container appears to still be running: killing"
603+
docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
604+
fi
605+
# See HADOOP-13951
606+
chmod -R u+rxw "${WORKSPACE}"
607+
'''
608+
dir ("${WORKDIR}") {
609+
deleteDir()
610+
}
611+
}
612+
}
613+
}
614+
}
477615
}
478616
}
479617
}

‎dev-support/docker/Dockerfile

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,13 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
112112
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \
113113
echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c -
114114

115+
FROM base_image AS openjdk17_download_image
116+
ENV OPENJDK17_URL 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'
117+
ENV OPENJDK17_SHA256 'a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378'
118+
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
119+
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \
120+
echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c -
121+
115122
##
116123
# build the final image
117124
#
@@ -160,6 +167,14 @@ RUN mkdir -p /usr/lib/jvm && \
160167
ln -s /usr/lib/jvm/java-11-adoptopenjdk /usr/lib/jvm/java-11 && \
161168
rm /tmp/adoptopenjdk11.tar.gz
162169

170+
# hadolint ignore=DL3010
171+
COPY --from=openjdk17_download_image /tmp/adoptopenjdk17.tar.gz /tmp/adoptopenjdk17.tar.gz
172+
RUN mkdir -p /usr/lib/jvm && \
173+
tar xzf /tmp/adoptopenjdk17.tar.gz -C /usr/lib/jvm && \
174+
ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk17.tar.gz | head -n1)")" /usr/lib/jvm/java-17-adoptopenjdk && \
175+
ln -s /usr/lib/jvm/java-17-adoptopenjdk /usr/lib/jvm/java-17 && \
176+
rm /tmp/adoptopenjdk17.tar.gz
177+
163178
# configure default environment for Yetus. Yetus in dockermode seems to require
164179
# these values to be specified here; the various --foo-path flags do not
165180
# propigate as expected, while these are honored.

0 commit comments

Comments
 (0)