Refactor Jenkinsfiles

Jenkinsfile

@@ -20,6 +20,32 @@
 // Jenkins pipeline
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
+
+/***
+ *      _____  _                          
+ *     |  __ \| |                         
+ *     | |__) | | ___  __ _ ___  ___      
+ *     |  ___/| |/ _ \/ _` / __|/ _ \     
+ *     | | | || |  __/ (_| \__ \  __/     
+ *     |_|_| ||_|\___|\__,_|___/\___|     
+ *      / _` |/ _ \                       
+ *     | (_| | (_) |_                     
+ *      \__,_|\___/| |                    
+ *      _ __   ___ | |_                   
+ *     | '_ \ / _ \| __|    _ _  __       
+ *     | | | | (_) | |_    | (_)/ _|      
+ *     |_|_|_|\___/_\__| __| |_| |_ _   _ 
+ *     | '_ ` _ \ / _ \ / _` | |  _| | | |
+ *     | | | | | | (_) | (_| | | | | |_| |
+ *     |_| |_| |_|\___/ \__,_|_|_|  \__, |
+ *                                   __/ |
+ *                                  |___/ 
+ *
+ * This file is about to be deprecated! See https://github.com/apache/incubator-mxnet/pull/13344
+ * for more details
+ */
+
+
 // mxnet libraries
 mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 

ci/Jenkinsfile_utils.groovy

@@ -65,8 +65,10 @@ def init_git_win() {
 // pack libraries for later use
 def pack_lib(name, libs, include_gcov_data = false) {
   sh """
+set +e
 echo "Packing ${libs} into ${name}"
 echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+return 0
 """
   stash includes: libs, name: name
 
@@ -82,32 +84,43 @@ def unpack_and_init(name, libs, include_gcov_data = false) {
   init_git()
   unstash name
   sh """
+set +e
 echo "Unpacked ${libs} from ${name}"
 echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+return 0
 """
   if (include_gcov_data) {
     // Restore GCNO files that are required for GCOV to operate during runtime
     unstash "${name}_gcov_data"
   }
 }
 
+def get_jenkins_master_url() {
+    return env.BUILD_URL.split('/')[2].split(':')[0]
+}
+
+def get_git_commit_hash() {
+  lastCommitMessage = sh (script: "git log -1 --pretty=%B", returnStdout: true)
+  lastCommitMessage = lastCommitMessage.trim()
+  if (lastCommitMessage.startsWith("Merge commit '") && lastCommitMessage.endsWith("' into HEAD")) {
+      // Merge commit applied by Jenkins, skip that commit
+      git_commit_hash = sh (script: "git rev-parse @~", returnStdout: true)
+  } else {
+      git_commit_hash = sh (script: "git rev-parse @", returnStdout: true)
+  }
+  return git_commit_hash
+}
+
 def publish_test_coverage() {
     // CodeCovs auto detection has trouble with our CIs PR validation due the merging strategy
-    lastCommitMessage = sh (script: "git log -1 --pretty=%B", returnStdout: true)
-    lastCommitMessage = lastCommitMessage.trim()
-    if (lastCommitMessage.startsWith("Merge commit '") && lastCommitMessage.endsWith("' into HEAD")) {
-        // Merge commit applied by Jenkins, skip that commit
-        GIT_COMMIT_HASH = sh (script: "git rev-parse @~", returnStdout: true)
-    } else {
-        GIT_COMMIT_HASH = sh (script: "git rev-parse @", returnStdout: true)
-    }
+    git_commit_hash = get_git_commit_hash()
 
     if (env.CHANGE_ID) {
       // PR execution
-      codecovArgs = "-B ${env.CHANGE_TARGET} -C ${GIT_COMMIT_HASH} -P ${env.CHANGE_ID}"
+      codecovArgs = "-B ${env.CHANGE_TARGET} -C ${git_commit_hash} -P ${env.CHANGE_ID}"
     } else {
       // Branch execution
-      codecovArgs = "-B ${env.BRANCH_NAME} -C ${GIT_COMMIT_HASH}"
+      codecovArgs = "-B ${env.BRANCH_NAME} -C ${git_commit_hash}"
     }
 
     // To make sure we never fail because test coverage reporting is not available
@@ -144,7 +157,67 @@ def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
   sh command
 }
 
+// Allow publishing to GitHub with a custom context (the status shown under a PR)
+// Credit to https://plugins.jenkins.io/github
+def get_repo_url() {
+  //item = Jenkins.instance.getItemByFullName("JOB_NAME")
+  //return item.getScm().getUserRemoteConfigs()[0].getUrl()
+  checkout scm
+  sh "git config --get remote.origin.url > .git/remote-url"
+  return readFile(".git/remote-url").trim()
+}
 
+def update_github_commit_status(state, message) {
+  node(NODE_LINUX_CPU) {
+    // NOTE: https://issues.jenkins-ci.org/browse/JENKINS-39482
+    //The GitHubCommitStatusSetter requires that the Git Server is defined under 
+    //*Manage Jenkins > Configure System > GitHub > GitHub Servers*. 
+    //Otherwise the GitHubCommitStatusSetter is not able to resolve the repository name 
+    //properly and you would see an empty list of repos:
+    //[Set GitHub commit status (universal)] PENDING on repos [] (sha:xxxxxxx) with context:test/mycontext
+    //See https://cwiki.apache.org/confluence/display/MXNET/Troubleshooting#Troubleshooting-GitHubcommit/PRstatusdoesnotgetpublished
+    repoUrl = get_repo_url()
+    commitSha = get_git_commit_hash()
+    context = get_github_context()
+
+    step([
+      $class: 'GitHubCommitStatusSetter',
+      reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl],
+      contextSource: [$class: "ManuallyEnteredCommitContextSource", context: context],
+      commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha],
+      statusBackrefSource: [$class: "ManuallyEnteredBackrefSource", backref: "${env.RUN_DISPLAY_URL}"],
+      errorHandlers: [[$class: 'ShallowAnyErrorHandler']],
+      statusResultSource: [
+        $class: 'ConditionalStatusResultSource',
+        results: [[$class: "AnyBuildResult", message: message, state: state]]
+      ]
+    ])
+  }
+}
+
+def get_github_context() {
+  // Since we use multi-branch pipelines, Jenkins appends the branch name to the job name
+  if (JOB_NAME.contains('/')) {
+    short_job_name = JOB_NAME.split('/')[0] 
+  } else {
+    short_job_name = JOB_NAME
+  }
+
+  return "ci/jenkins/${short_job_name}"
+}
+
+def parallel_stage(stage_name, steps) {
+    // Allow to pass an array of steps that will be executed in parallel in a stage
+    new_map = [:]
+
+    for (def step in steps) {
+        new_map = new_map << step
+    }
+
+    stage(stage_name) {
+      parallel new_map
+    }
+}
 
 def assign_node_labels(args) {
   NODE_LINUX_CPU = args.linux_cpu
@@ -164,15 +237,18 @@ def main_wrapper(args) {
   // assign any caught errors here
   err = null
   try {
+    update_github_commit_status('PENDING', 'Job has been enqueued')
     args['core_logic']()
 
     // set build status to success at the end
     currentBuild.result = "SUCCESS"
+    update_github_commit_status('SUCCESS', 'Job succeeded')
   } catch (caughtError) {
     node(NODE_LINUX_CPU) {
       sh "echo caught ${caughtError}"
       err = caughtError
       currentBuild.result = "FAILURE"
+      update_github_commit_status('FAILURE', 'Job failed')
     }
   } finally {
     node(NODE_LINUX_CPU) {