Skip to content

Commit

Permalink
HADOOP-17807. Use separate src dir for platform builds (#3210)
Browse files Browse the repository at this point in the history
  • Loading branch information
@GauthamBanasandra
GauthamBanasandra authored Jul 26, 2021
1 parent d710ec8 commit 97c88c9
Show file tree
Hide file tree
Showing 2 changed files with 194 additions and 69 deletions.
195 changes: 139 additions & 56 deletions dev-support/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ pipeline {
}

environment {
SOURCEDIR = 'src'
// will also need to change notification section below
PATCHDIR = 'out'
YETUS='yetus'
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
Expand All @@ -56,13 +53,44 @@ pipeline {
}
}

// Setup codebase so that each platform's build happens in its own exclusive copy of the
// codebase.
// Primarily because YETUS messes up the git branch information and affects the subsequent
// optional stages after the first one.
stage ('setup sources') {
steps {
dir("${WORKSPACE}/centos-8") {
sh '''#!/usr/bin/env bash
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
'''
}

dir("${WORKSPACE}/debian-10") {
sh '''#!/usr/bin/env bash
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
'''
}

dir("${WORKSPACE}/ubuntu-focal") {
sh '''#!/usr/bin/env bash
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal
'''
}
}
}

// This is an optional stage which runs only when there's a change in
// C++/C++ build/platform.
// This stage serves as a means of cross platform validation, which is
// really needed to ensure that any C++ related/platform change doesn't
// break the Hadoop build on Centos 8.
stage ('precommit-run Centos 8') {
environment {
SOURCEDIR = "${WORKSPACE}/centos-8/src"
PATCHDIR = "${WORKSPACE}/centos-8/out"
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
IS_OPTIONAL = 1
}
Expand All @@ -78,10 +106,32 @@ pipeline {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
'''
}
}

post {
// Since this is an optional platform, we want to copy the artifacts
// and archive it only if the build fails, to help with debugging.
failure {
sh '''#!/usr/bin/env bash
cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
'''
archiveArtifacts "out/**"
}

cleanup() {
script {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
'''
}
}
}
}

// This is an optional stage which runs only when there's a change in
Expand All @@ -91,6 +141,8 @@ pipeline {
// break the Hadoop build on Debian 10.
stage ('precommit-run Debian 10') {
environment {
SOURCEDIR = "${WORKSPACE}/debian-10/src"
PATCHDIR = "${WORKSPACE}/debian-10/out"
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
IS_OPTIONAL = 1
}
Expand All @@ -106,16 +158,40 @@ pipeline {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
'''
}
}

post {
// Since this is an optional platform, we want to copy the artifacts
// and archive it only if the build fails, to help with debugging.
failure {
sh '''#!/usr/bin/env bash
cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
'''
archiveArtifacts "out/**"
}

cleanup() {
script {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
'''
}
}
}
}

// We want to use Ubuntu Focal as our main CI and thus, this stage
// isn't optional (runs for all the PRs).
stage ('precommit-run Ubuntu focal') {
environment {
SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src"
PATCHDIR = "${WORKSPACE}/ubuntu-focal/out"
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
IS_OPTIONAL = 0
}
Expand All @@ -131,69 +207,76 @@ pipeline {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
'''
}
}
}

}
post {
always {
script {
// Publish status if it was missed (YETUS-1059)
withCredentials(
[usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER')]) {
sh '''#!/usr/bin/env bash
post {
always {
script {
// Publish status if it was missed (YETUS-1059)
withCredentials(
[usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER')]) {
sh '''#!/usr/bin/env bash
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
'''
}
# Copy the artifacts of Ubuntu focal build to workspace
cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}"
# Send Github status
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery
'''
}

// Yetus output
archiveArtifacts "${env.PATCHDIR}/**"
// Publish the HTML report so that it can be looked at
// Has to be relative to WORKSPACE.
publishHTML (target: [
allowMissing: true,
keepAll: true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE
reportDir: "${env.PATCHDIR}",
reportFiles: 'report.html',
reportName: 'Yetus Report'
])
// Publish JUnit results
try {
junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
} catch(e) {
echo 'junit processing: ' + e.toString()
// YETUS output
archiveArtifacts "out/**"

// Publish the HTML report so that it can be looked at
// Has to be relative to WORKSPACE.
publishHTML (target: [
allowMissing: true,
keepAll: true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE
reportDir: "out",
reportFiles: 'report.html',
reportName: 'Yetus Report'
])

// Publish JUnit results
try {
junit "${SOURCEDIR}/**/target/surefire-reports/*.xml"
} catch(e) {
echo 'junit processing: ' + e.toString()
}
}
}

cleanup() {
script {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
'''
}
}
}
}
}
}

post {
// Jenkins pipeline jobs fill slaves on PRs without this :(
cleanup() {
script {
sh '''
# See YETUS-764
if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
echo "test-patch process appears to still be running: killing"
kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
sleep 10
fi
if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
echo "test-patch container appears to still be running: killing"
docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
fi
# See HADOOP-13951
chmod -R u+rxw "${WORKSPACE}"
'''
sh '''#!/usr/bin/env bash
# See HADOOP-13951
chmod -R u+rxw "${WORKSPACE}"
'''
deleteDir()
}
}
Expand Down
68 changes: 55 additions & 13 deletions dev-support/jenkins.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,10 @@ function run_ci() {
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"

# this must be clean for every run
if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
rm -rf "${WORKSPACE:?}/${PATCHDIR}"
if [[ -d "${PATCHDIR}" ]]; then
rm -rf "${PATCHDIR:?}"
fi
mkdir -p "${WORKSPACE}/${PATCHDIR}"
mkdir -p "${PATCHDIR}"

# if given a JIRA issue, process it. If CHANGE_URL is set
# (e.g., Github Branch Source plugin), process it.
Expand All @@ -128,23 +128,23 @@ function run_ci() {
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
elif [[ -z "${CHANGE_URL}" ]]; then
echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html"
echo "Full build skipped" >"${PATCHDIR}/report.html"
exit 0
fi

YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
YETUS_ARGS+=("--patch-dir=${PATCHDIR}")

# where the source is located
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
YETUS_ARGS+=("--basedir=${SOURCEDIR}")

# our project defaults come from a personality file
YETUS_ARGS+=("--project=hadoop")
YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/bin/hadoop.sh")

# lots of different output formats
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")

# enable writing back to Github
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
Expand Down Expand Up @@ -206,7 +206,49 @@ function run_ci() {
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
}

# Check if the CI needs to be run, if so, do so :)
if check_ci_run; then
run_ci
## @description Cleans up the processes started by YETUS
function cleanup_ci_proc() {
# See YETUS-764
if [ -f "${PATCHDIR}/pidfile.txt" ]; then
echo "test-patch process appears to still be running: killing"
kill "$(cat "${PATCHDIR}/pidfile.txt")" || true
sleep 10
fi
if [ -f "${PATCHDIR}/cidfile.txt" ]; then
echo "test-patch container appears to still be running: killing"
docker kill "$(cat "${PATCHDIR}/cidfile.txt")" || true
fi
}

## @description Invokes github_status_recovery in YETUS's precommit
function github_status_recovery() {
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${EXTRA_ARGS}" || true
}

if [ -z "$1" ]; then
echo "Must specify an argument for jenkins.sh"
echo "run_ci - Runs the CI based on platform image as defined by DOCKERFILE"
echo "cleanup_ci_proc - Cleans up the processes spawned for running the CI"
echo "github_status_recovery - Sends Github status (refer to YETUS precommit for more details)"
exit 1
fi

# Process arguments to jenkins.sh
if [ "$1" == "run_ci" ]; then
# Check if the CI needs to be run, if so, do so :)
if check_ci_run; then
run_ci
else
echo "No C++ file/C++ build/platform changes found, will not run CI"
fi
elif [ "$1" == "cleanup_ci_proc" ]; then
cleanup_ci_proc
elif [ "$1" == "github_status_recovery" ]; then
github_status_recovery
else
echo "Don't know how to process $1"
exit 1
fi

0 comments on commit 97c88c9

Please sign in to comment.