Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix Jenkins success reporting #2302

Merged
merged 3 commits into from
Feb 10, 2024
Merged

Fix Jenkins success reporting #2302

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d5384bc
moved post back to the end of the main stages block
Feb 9, 2024
b0e416b
just to experments
Feb 9, 2024
91d35e1
paired down errounous info in comment about running expirement
Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 77 additions & 77 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ pipeline {
machine = 'none'
for (label in pullRequest.labels) {
echo "Label: ${label}"
if ((label.matches("CI-Hera-Ready"))) {
if ((label.matches('CI-Hera-Ready'))) {
machine = 'hera'
} else if ((label.matches("CI-Orion-Ready"))) {
} else if ((label.matches('CI-Orion-Ready'))) {
machine = 'orion'
} else if ((label.matches("CI-Hercules-Ready"))) {
} else if ((label.matches('CI-Hercules-Ready'))) {
machine = 'hercules'
}
} // createing a second machine varible with first letter capital
// because the first letter of the machine name is captitalized in the GitHub labels
} // createing a second machine varible with first letter capital
// because the first letter of the machine name is captitalized in the GitHub labels
Machine = machine[0].toUpperCase() + machine.substring(1)
}
}
Expand All @@ -42,17 +42,17 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}/TESTDIR"
commonworkspace = "${WORKSPACE}"
sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
sh(script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
pullRequest.addLabel("CI-${Machine}-Building")
if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) {
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
pullRequest.removeLabel("CI-${Machine}-Ready")
}
}
}
}
}

stage('Build System') {
matrix {
Expand All @@ -61,121 +61,82 @@ pipeline {
// throttle(['global_matrix_build'])
//}
axes {
axis {
name "system"
values "gfs", "gefs"
axis {
name 'system'
values 'gfs', 'gefs'
}
}
stages {
stage("build system") {
stage('build system') {
steps {
script {
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
sh(script: "mkdir -p ${HOMEgfs}", returnStatus: true)
ws(HOMEgfs) {
env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment
if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
} else {
checkout scm
sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive', returnStatus: true)
def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
def build_args_list = builds_file['builds']
def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
dir("${HOMEgfs}/sorc") {
sh( script: "${build_args}", returnStatus: true)
sh( script: "./link_workflow.sh", returnStatus: true)
sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
sh(script: "${build_args}", returnStatus: true)
sh(script: './link_workflow.sh', returnStatus: true)
sh(script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
}
}
if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
pullRequest.removeLabel("CI-${Machine}-Building")
}
pullRequest.addLabel("CI-${Machine}-Running")
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
pullRequest.removeLabel("CI-${Machine}-Building")
}
pullRequest.addLabel("CI-${Machine}-Running")
}
}
}
}
}
}
}

stage('Run Tests') {
matrix {
agent { label "${machine}-emc" }
axes {
axis {
name "Case"
values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" // TODO add dynamic list of cases from env vars (needs addtional plugins)
name 'Case'
values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' // TODO add dynamic list of cases from env vars (needs addtional plugins)
}
}
stages {
stage('Create Experiment') {
steps {
steps {
script {
sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
system = yaml_case.experiment.system
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis
env.RUNTESTS = "${HOME}/RUNTESTS"
sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
}
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
}
}
}
stage('Run Experiments') {
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
ws(HOMEgfs) {
pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
try {
sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
} catch (Exception e) {
pullRequest.comment("**FAILURE** running experiments: ${Case} on ${Machine}")
error("Failed to run experiments ${Case} on ${Machine}")
}
pullRequest.comment("**SUCCESS** running experiments: ${Case} on ${Machine}")
}
}
}
post {
always {
script {
ws (HOMEgfs) {
for (label in pullRequest.labels) {
if (label.contains("${Machine}")) {
pullRequest.removeLabel(label)
}
}
}
}
}
success {
script {
ws (HOMEgfs) {
pullRequest.addLabel("CI-${Machine}-Passed")
def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
}
}
}
failure {
script {
ws (HOMEgfs) {
pullRequest.addLabel("CI-${Machine}-Failed")
def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
if (fileExists('${HOME}/RUNTESTS/ci.log')) {
def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
fileContent.eachLine { line ->
if( line.contains(".log")) {
archiveArtifacts artifacts: "${line}", fingerprint: true
}
}
}
pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
error("Failed to run experiments ${Case} on ${Machine}")
}
pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
}
}
}
Expand All @@ -185,4 +146,43 @@ pipeline {
}
}

post {
always {
script {
if(env.CHANGE_ID) {
for (label in pullRequest.labels) {
if (label.contains("${Machine}")) {
pullRequest.removeLabel(label)
}
}
}
}
}
success {
script {
if(env.CHANGE_ID) {
pullRequest.addLabel("CI-${Machine}-Passed")
def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
}
}
}
failure {
script {
if(env.CHANGE_ID) {
pullRequest.addLabel("CI-${Machine}-Failed")
def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
}
if (fileExists('${HOME}/RUNTESTS/ci.log')) {
def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
fileContent.eachLine { line ->
if (line.contains('.log')) {
archiveArtifacts artifacts: "${line}", fingerprint: true
}
}
}
}
}
}
}
Loading