Parameterised build pipeline generator

Users can now generate their own pipelines using this script from their own repositories to where they want to generate them.
Signed-off-by: Morgan Davies <morgandavies2020@gmail.com>

build-farm/set-platform-specific-configurations.sh

@@ -26,4 +26,9 @@ fi
 export VARIANT_ARG="--build-variant ${VARIANT}"
 
 # shellcheck disable=SC1091,SC1090
-source "$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh"
+if [ ! -z "$PLATFORM_CONFIG_PATH" ]
+then
+    source "$SCRIPT_DIR/${PLATFORM_CONFIG_PATH}"
+else
+    source "$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh"
+fi

pipelines/build/common/build_base_file.groovy

@@ -71,6 +71,8 @@ class Builder implements Serializable {
 
         def dockerNode = getDockerNode(platformConfig, variant)
 
+        def platformSpecificConfigPath = getPlatformSpecificConfigPath(platformConfig)
+
         def buildArgs = getBuildArgs(platformConfig, variant)
 
         if (additionalBuildArgs) {
@@ -98,6 +100,7 @@ class Builder implements Serializable {
                 DOCKER_IMAGE: dockerImage,
                 DOCKER_FILE: dockerFile,
                 DOCKER_NODE: dockerNode,
+                PLATFORM_CONFIG_PATH: platformSpecificConfigPath,
                 CONFIGURE_ARGS: getConfigureArgs(platformConfig, additionalConfigureArgs, variant),
                 OVERRIDE_FILE_NAME_VERSION: overrideFileNameVersion,
                 ADDITIONAL_FILE_NAME_TAG: platformConfig.additionalFileNameTag as String,
@@ -227,6 +230,14 @@ class Builder implements Serializable {
         return dockerNodeValue
     }
 
+    def getPlatformSpecificConfigPath(Map<String, ?> configuration) {
+        def platformSpecificConfigPath = ""
+        if (configuration.containsKey("platformSpecificConfigPath")) {
+            platformSpecificConfigPath = configuration.platformSpecificConfigPath
+        }
+        return platformSpecificConfigPath
+    }
+
     /**
      * Builds up a node param string that defines what nodes are eligible to run the given job
      * @param configuration

pipelines/build/common/config_regeneration.groovy

@@ -130,6 +130,14 @@ class Regeneration implements Serializable {
         return dockerNodeValue
     }
 
+    def getPlatformSpecificConfigPath(Map<String, ?> configuration) {
+        def platformSpecificConfigPath = ""
+        if (configuration.containsKey("platformSpecificConfigPath")) {
+            platformSpecificConfigPath = configuration.platformSpecificConfigPath
+        }
+        return platformSpecificConfigPath
+    }
+
     /**
     * Builds up a node param string that defines what nodes are eligible to run the given job. Used as a placeholder since the pipelines overwrite this.
     * @param configuration
@@ -253,6 +261,8 @@ class Regeneration implements Serializable {
 
             def dockerNode = getDockerNode(platformConfig, variant)
 
+            def platformSpecificConfigPath = getPlatformSpecificConfigPath(platformConfig)
+
             def buildArgs = getBuildArgs(platformConfig, variant)
 
             def testList = getTestList(platformConfig)
@@ -270,6 +280,7 @@ class Regeneration implements Serializable {
                 DOCKER_IMAGE: dockerImage,
                 DOCKER_FILE: dockerFile,
                 DOCKER_NODE: dockerNode,
+                PLATFORM_CONFIG_PATH: platformSpecificConfigPath,
                 CONFIGURE_ARGS: getConfigureArgs(platformConfig, variant),
                 OVERRIDE_FILE_NAME_VERSION: "",
                 ADDITIONAL_FILE_NAME_TAG: platformConfig.additionalFileNameTag as String,

pipelines/build/common/create_job_from_template.groovy

@@ -74,6 +74,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
                 <dt><strong>CODEBUILD</strong></dt><dd>Use a dynamic codebuild machine if no other machine is available</dd>
                 <dt><strong>DOCKER_IMAGE</strong></dt><dd>Use a docker build environment</dd>
                 <dt><strong>DOCKER_FILE</strong></dt><dd>Relative path to a dockerfile to be built and used on top of the DOCKER_IMAGE</dd>
+                <dt><strong>PLATFORM_CONFIG_PATH</strong></dt><dd>Relative path to the platform specific configuration for this paticular OS</dd>
                 <dt><strong>CONFIGURE_ARGS</strong></dt><dd>Arguments for ./configure</dd>
                 <dt><strong>OVERRIDE_FILE_NAME_VERSION</strong></dt><dd>Set the version string on the file name</dd>
                 <dt><strong>RELEASE</strong></dt><dd>Is this build a release</dd>

pipelines/build/openjdk8_pipeline.groovy

@@ -20,11 +20,25 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load BASE_FILE_PATH. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    def configureBuild = null
+    if ("$BASE_FILE_PATH" != "") {
+        configureBuild = load "${WORKSPACE}/${BASE_FILE_PATH}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load BUILD_CONFIG_FILE_PATH. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$BUILD_CONFIG_FILE_PATH" != "") {
+        buildConfigurations = load "${WORKSPACE}/${BUILD_CONFIG_FILE_PATH}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,

pipelines/build/regeneration/build_pipeline_generator.groovy

@@ -1,32 +1,67 @@
+import java.nio.file.NoSuchFileException
+
 node('master') {
-  def retiredVersions = [9, 10, 12, 13, 14]
+  def retiredVersions = [9, 10, 12, 13]
+  def generatedPipelines = []
+
+  // Load gitUri and gitBranch. These determine where we will be pulling configs from.
+  def repoUri = "$REPOSITORY_URL" != "" ? REPOSITORY_URL : "https://github.com/AdoptOpenJDK/openjdk-build.git"
+  def repoBranch = "$REPOSITORY_BRANCH" != "" ? REPOSITORY_BRANCH : "master"
+
+  // Load jobRoot. This is where the openjdkxx-pipeline jobs will be created.
+  def jobRoot = "$JOB_ROOT" != "" ? JOB_ROOT : "build-scripts"
+
+  // Load scriptFolderPath. This is the folder where the openjdkxx-pipeline.groovy code is located compared to the repository root. These are the top level pipeline jobs.
+  def scriptFolderPath = "$SCRIPT_FOLDER_PATH" != "" ? SCRIPT_FOLDER_PATH : "pipelines/build"
+
+  // Load nightlyFolderPath. This is the folder where the jdkxx.groovy code is located compared to the repository root. These define what the default set of nightlies will be.
+  def nightlyFolderPath = "$NIGHTLY_FOLDER_PATH" != "" ? NIGHTLY_FOLDER_PATH : "pipelines/jobs/configurations"
+
+  // Load jobTemplatePath. This is where the pipeline_job_template.groovy code is located compared to the repository root. This actually sets up the pipeline job using the parameters above.
+  def jobTemplatePath = "$JOB_TEMPLATE_PATH" != "" ? JOB_TEMPLATE_PATH : "pipelines/jobs/pipeline_job_template.groovy"
+
+  println "[INFO] Running generator script with the following configuration:"
+  println "REPOSITORY_URL = $repoUri"
+  println "REPOSITORY_BRANCH = $repoBranch"
+  println "JOB_ROOT = $jobRoot"
+  println "SCRIPT_FOLDER_PATH = $scriptFolderPath"
+  println "NIGHTLY_FOLDER_PATH = $nightlyFolderPath"
+  println "JOB_TEMPLATE_PATH = $jobTemplatePath"
+  println "ENABLE_PIPELINE_SCHEDULE = $ENABLE_PIPELINE_SCHEDULE"
 
   (8..30).each({javaVersion -> 
 
     if (retiredVersions.contains(javaVersion)) {
       println "[INFO] $javaVersion is a retired version that isn't built anymore. Skipping generation..."
       return
     }
-    
+
     def config = [
       TEST                : false,
-      GIT_URL             : "https://github.com/AdoptOpenJDK/openjdk-build.git",
-      BRANCH              : "master",
-      BUILD_FOLDER        : "build-scripts",
+      GIT_URL             : repoUri,
+      BRANCH              : repoBranch,
+      BUILD_FOLDER        : jobRoot,
       JOB_NAME            : "openjdk${javaVersion}-pipeline",
-      SCRIPT              : "pipelines/build/openjdk${javaVersion}_pipeline.groovy",
+      SCRIPT              : "${scriptFolderPath}/openjdk${javaVersion}_pipeline.groovy",
       disableJob          : false,
       triggerSchedule     : ""
     ];
-    checkout([$class: 'GitSCM', userRemoteConfigs: [[url: config.GIT_URL]]])
+
+    checkout(
+      [
+        $class: 'GitSCM',
+        branches: [[name: config.BRANCH ]],
+        userRemoteConfigs: [[ url: config.GIT_URL ]]
+      ]
+    )
 
     def target;
     try {
-      target = load "${WORKSPACE}/pipelines/jobs/configurations/jdk${javaVersion}u.groovy"
-    } catch(Exception e) {
+      target = load "${WORKSPACE}/${nightlyFolderPath}/jdk${javaVersion}u.groovy"
+    } catch(NoSuchFileException e) {
       try {
-        target = load "${WORKSPACE}/pipelines/jobs/configurations/jdk${javaVersion}.groovy"
-      } catch(Exception e2) {
+        target = load "${WORKSPACE}/${nightlyFolderPath}/jdk${javaVersion}.groovy"
+      } catch(NoSuchFileException e2) {
         println "[WARNING] No config found for JDK${javaVersion}"
         return
       }
@@ -43,7 +78,8 @@ node('master') {
 
     println "[INFO] JDK${javaVersion}: disableJob = ${config.disableJob}"
 
-    if (Boolean.parseBoolean(enablePipelineSchedule) == true) {
+    // Load ENABLE_PIPELINE_SCHEDULE. This determines whether the jobs should run automatically or not.
+    if (Boolean.parseBoolean(ENABLE_PIPELINE_SCHEDULE) == true) {
       try {
         config.triggerSchedule = target.triggerSchedule
       } catch (Exception ex) {
@@ -53,7 +89,18 @@ node('master') {
 
     println "[INFO] JDK${javaVersion}: triggerSchedule = ${config.triggerSchedule}"
 
-    def create = jobDsl targets: "pipelines/jobs/pipeline_job_template.groovy", ignoreExisting: false, additionalParameters: config
+    def create = jobDsl targets: jobTemplatePath, ignoreExisting: false, additionalParameters: config
     target.disableJob = false
+
+    generatedPipelines.add(javaVersion)
   })
+
+  // Fail if nothing was generated
+  if (generatedPipelines == []) {
+    throw new Exception("[ERROR] NO PIPELINES WERE GENERATED!")
+  } else {
+    println "[SUCCESS] THE FOLLOWING PIPELINES WERE GENERATED IN THE $JOB_ROOT FOLDER"
+    println generatedPipelines
+  }
+
 }

pipelines/build/regeneration/jdk8_regeneration_pipeline.groovy

@@ -21,6 +21,10 @@ node ("master") {
     def scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
 
+    // Load gitUri and gitBranch. These determine where we will be pulling configs from.
+    def repoUri = "$REPOSITORY_URL" != "" ? REPOSITORY_URL : null
+    def repoBranch = "$REPOSITORY_BRANCH" != "" ? REPOSITORY_BRANCH : null
+
     // Load buildConfigurations from config file. This is what the nightlies & releases use to setup their downstream jobs
     def buildConfigurations = null
     def buildConfigPath = "${WORKSPACE}/pipelines/jobs/configurations/${javaVersion}_pipeline_config.groovy"
@@ -64,6 +68,8 @@ node ("master") {
 
     println "[INFO] Running regeneration script with the following configuration:"
     println "VERSION: $javaVersion"
+    println "REPOSITORY URL: $repoUri"
+    println "REPOSITORY BRANCH: $repoBranch"
     println "BUILD CONFIGURATIONS: $buildConfigurations"
     println "JOBS TO GENERATE: $targetConfigurations"
     println "JOB ROOT: $jobRoot"
@@ -91,8 +97,8 @@ node ("master") {
           currentBuild,
           this,
           jobRoot,
-          null,
-          null,
+          repoUri,
+          repoBranch,
           jenkinsBuildRoot,
           jenkinsUsername,
           jenkinsToken
@@ -110,8 +116,8 @@ node ("master") {
         currentBuild,
         this,
         jobRoot,
-        null,
-        null,
+        repoUri,
+        repoBranch,
         jenkinsBuildRoot,
         null,
         null

pipelines/library/src/common/IndividualBuildConfig.groovy

@@ -16,6 +16,7 @@ class IndividualBuildConfig implements Serializable {
     final String DOCKER_IMAGE
     final String DOCKER_FILE
     final String DOCKER_NODE
+    final String PLATFORM_CONFIG_PATH
     final String CONFIGURE_ARGS
     final String OVERRIDE_FILE_NAME_VERSION
     final String ADDITIONAL_FILE_NAME_TAG
@@ -52,6 +53,7 @@ class IndividualBuildConfig implements Serializable {
         DOCKER_IMAGE = map.get("DOCKER_IMAGE")
         DOCKER_FILE = map.get("DOCKER_FILE")
         DOCKER_NODE = map.get("DOCKER_NODE")
+        PLATFORM_CONFIG_PATH = map.get("PLATFORM_CONFIG_PATH")
         CONFIGURE_ARGS = map.get("CONFIGURE_ARGS")
         OVERRIDE_FILE_NAME_VERSION = map.get("OVERRIDE_FILE_NAME_VERSION")
         ADDITIONAL_FILE_NAME_TAG = map.get("ADDITIONAL_FILE_NAME_TAG")
@@ -93,6 +95,7 @@ class IndividualBuildConfig implements Serializable {
                 DOCKER_IMAGE              : DOCKER_IMAGE,
                 DOCKER_FILE               : DOCKER_FILE,
                 DOCKER_NODE               : DOCKER_NODE,
+                PLATFORM_CONFIG_PATH      : PLATFORM_CONFIG_PATH,
                 CONFIGURE_ARGS            : CONFIGURE_ARGS,
                 OVERRIDE_FILE_NAME_VERSION: OVERRIDE_FILE_NAME_VERSION,
                 ADDITIONAL_FILE_NAME_TAG  : ADDITIONAL_FILE_NAME_TAG,