Skip to content

Commit bffae99

Browse files
Apache9Apache9
authored
HBASE-27293 Remove jenkins and personality scripts support for 1.x (#4690)
Signed-off-by: GeorryHuang <huangzhuoyue@apache.org>
1 parent 2c3abae commit bffae99

File tree

2 files changed

+43
-251
lines changed

2 files changed

+43
-251
lines changed

‎dev-support/Jenkinsfile

Lines changed: 35 additions & 163 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ pipeline {
3434
YETUS_RELEASE = '0.12.0'
3535
// where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
3636
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
37-
OUTPUT_DIR_RELATIVE_JDK7 = 'output-jdk7'
3837
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
3938
OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3'
4039
OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3'
@@ -186,7 +185,6 @@ pipeline {
186185
// stash with given name for all tests we might run, so that we can unstash all of them even if
187186
// we skip some due to e.g. branch-specific JDK or Hadoop support
188187
stash name: 'general-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_GENERAL}/doesn't-match"
189-
stash name: 'jdk7-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK7}/doesn't-match"
190188
stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match"
191189
stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match"
192190
stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match"
@@ -296,124 +294,14 @@ pipeline {
296294
}
297295
}
298296
}
299-
stage ('yetus jdk7 checks') {
300-
agent {
301-
node {
302-
label 'hbase'
303-
}
304-
}
305-
when {
306-
branch 'branch-1*'
307-
}
308-
environment {
309-
BASEDIR = "${env.WORKSPACE}/component"
310-
TESTS = "${env.DEEP_CHECKS}"
311-
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
312-
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
313-
SET_JAVA_HOME = "/usr/lib/jvm/java-7"
314-
}
315-
steps {
316-
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
317-
sh '''#!/usr/bin/env bash
318-
set -e
319-
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
320-
echo '(x) {color:red}-1 jdk7 checks{color}' >"${OUTPUT_DIR}/commentfile"
321-
echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
322-
'''
323-
unstash 'yetus'
324-
dir('component') {
325-
checkout scm
326-
}
327-
sh '''#!/usr/bin/env bash
328-
set -e
329-
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
330-
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
331-
echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
332-
ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
333-
'''
334-
script {
335-
def ret = sh(
336-
returnStatus: true,
337-
script: '''#!/usr/bin/env bash
338-
set -e
339-
declare -i status=0
340-
if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
341-
echo '(/) {color:green}+1 jdk7 checks{color}' > "${OUTPUT_DIR}/commentfile"
342-
else
343-
echo '(x) {color:red}-1 jdk7 checks{color}' > "${OUTPUT_DIR}/commentfile"
344-
status=1
345-
fi
346-
echo "-- For more information [see jdk7 report|${BUILD_URL}/JDK7_20Nightly_20Build_20Report/]" >> "${OUTPUT_DIR}/commentfile"
347-
exit "${status}"
348-
'''
349-
)
350-
if (ret != 0) {
351-
// mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
352-
// test output. See HBASE-26339 for more details.
353-
currentBuild.result = 'UNSTABLE'
354-
}
355-
}
356-
}
357-
post {
358-
always {
359-
stash name: 'jdk7-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
360-
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
361-
// zip surefire reports.
362-
sh '''#!/bin/bash -e
363-
if [ -d "${OUTPUT_DIR}/archiver" ]; then
364-
count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
365-
if [[ 0 -ne ${count} ]]; then
366-
echo "zipping ${count} archived files"
367-
zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
368-
else
369-
echo "No archived files, skipping compressing."
370-
fi
371-
else
372-
echo "No archiver directory, skipping compressing."
373-
fi
374-
'''
375-
sshPublisher(publishers: [
376-
sshPublisherDesc(configName: 'Nightlies',
377-
transfers: [
378-
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
379-
sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip"
380-
)
381-
]
382-
)
383-
])
384-
// remove the big test logs zip file, store the nightlies url in test_logs.html
385-
sh '''#!/bin/bash -e
386-
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
387-
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
388-
rm -rf "${OUTPUT_DIR}/test_logs.zip"
389-
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
390-
else
391-
echo "No test_logs.zip, skipping"
392-
fi
393-
'''
394-
// Has to be relative to WORKSPACE.
395-
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
396-
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
397-
publishHTML target: [
398-
allowMissing : true,
399-
keepAll : true,
400-
alwaysLinkToLastBuild: true,
401-
// Has to be relative to WORKSPACE.
402-
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
403-
reportFiles : 'console-report.html',
404-
reportName : 'JDK7 Nightly Build Report'
405-
]
406-
}
407-
}
408-
}
409297
stage ('yetus jdk8 hadoop2 checks') {
410298
agent {
411299
node {
412300
label 'hbase'
413301
}
414302
}
415303
when {
416-
anyOf { branch 'branch-1*'; branch 'branch-2*' }
304+
branch 'branch-2*'
417305
}
418306
environment {
419307
BASEDIR = "${env.WORKSPACE}/component"
@@ -522,11 +410,6 @@ pipeline {
522410
label 'hbase'
523411
}
524412
}
525-
when {
526-
not {
527-
branch 'branch-1*'
528-
}
529-
}
530413
environment {
531414
BASEDIR = "${env.WORKSPACE}/component"
532415
TESTS = "${env.DEEP_CHECKS}"
@@ -636,11 +519,6 @@ pipeline {
636519
label 'hbase'
637520
}
638521
}
639-
when {
640-
not {
641-
branch 'branch-1*'
642-
}
643-
}
644522
environment {
645523
BASEDIR = "${env.WORKSPACE}/component"
646524
TESTS = "${env.DEEP_CHECKS}"
@@ -817,7 +695,7 @@ pipeline {
817695
'''
818696
unstash 'hadoop-2'
819697
sh '''#!/bin/bash -xe
820-
if [[ "${BRANCH}" = branch-2* ]] || [[ "${BRANCH}" = branch-1* ]]; then
698+
if [[ "${BRANCH}" = branch-2* ]]; then
821699
echo "Attempting to use run an instance on top of Hadoop 2."
822700
artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
823701
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
@@ -841,44 +719,40 @@ pipeline {
841719
'''
842720
unstash 'hadoop-3'
843721
sh '''#!/bin/bash -e
844-
if [[ "${BRANCH}" = branch-1* ]]; then
845-
echo "Skipping to run against Hadoop 3 for branch ${BRANCH}"
846-
else
847-
echo "Attempting to use run an instance on top of Hadoop 3."
848-
artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
849-
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
850-
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
851-
--single-process \
852-
--working-dir output-integration/hadoop-3 \
853-
--hbase-client-install hbase-client \
854-
hbase-install \
855-
hadoop-3/bin/hadoop \
856-
hadoop-3/share/hadoop/yarn/timelineservice \
857-
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
858-
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
859-
hadoop-3/bin/mapred \
860-
>output-integration/hadoop-3.log 2>&1 ; then
861-
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
862-
exit 2
863-
fi
864-
echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
865-
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
866-
--single-process \
867-
--hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
868-
--working-dir output-integration/hadoop-3-shaded \
869-
--hbase-client-install hbase-client \
870-
hbase-install \
871-
hadoop-3/bin/hadoop \
872-
hadoop-3/share/hadoop/yarn/timelineservice \
873-
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
874-
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
875-
hadoop-3/bin/mapred \
876-
>output-integration/hadoop-3-shaded.log 2>&1 ; then
877-
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
878-
exit 2
879-
fi
880-
echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
722+
echo "Attempting to use run an instance on top of Hadoop 3."
723+
artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
724+
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
725+
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
726+
--single-process \
727+
--working-dir output-integration/hadoop-3 \
728+
--hbase-client-install hbase-client \
729+
hbase-install \
730+
hadoop-3/bin/hadoop \
731+
hadoop-3/share/hadoop/yarn/timelineservice \
732+
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
733+
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
734+
hadoop-3/bin/mapred \
735+
>output-integration/hadoop-3.log 2>&1 ; then
736+
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
737+
exit 2
738+
fi
739+
echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
740+
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
741+
--single-process \
742+
--hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
743+
--working-dir output-integration/hadoop-3-shaded \
744+
--hbase-client-install hbase-client \
745+
hbase-install \
746+
hadoop-3/bin/hadoop \
747+
hadoop-3/share/hadoop/yarn/timelineservice \
748+
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
749+
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
750+
hadoop-3/bin/mapred \
751+
>output-integration/hadoop-3-shaded.log 2>&1 ; then
752+
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
753+
exit 2
881754
fi
755+
echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
882756
'''
883757
}
884758
post {
@@ -919,14 +793,12 @@ pipeline {
919793
script {
920794
try {
921795
unstash 'general-result'
922-
unstash 'jdk7-result'
923796
unstash 'jdk8-hadoop2-result'
924797
unstash 'jdk8-hadoop3-result'
925798
unstash 'jdk11-hadoop3-result'
926799
unstash 'srctarball-result'
927800
sh "printenv"
928801
def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile",
929-
"${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile",
930802
"${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile",
931803
"${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile",
932804
"${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile",

0 commit comments

Comments
 (0)