Parameterized Generators and Pipelines

build-farm/set-platform-specific-configurations.sh

@@ -26,4 +26,9 @@ fi
 export VARIANT_ARG="--build-variant ${VARIANT}"
 
 # shellcheck disable=SC1091,SC1090
-source "$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh"
+if [ ! -z "$PLATFORM_CONFIG_PATH" ]
+then
+    source "$SCRIPT_DIR/${PLATFORM_CONFIG_PATH}"
+else
+    source "$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh"
+fi

pipelines/build/common/build_base_file.groovy

@@ -71,6 +71,8 @@ class Builder implements Serializable {
 
         def dockerNode = getDockerNode(platformConfig, variant)
 
+        def platformSpecificConfigPath = getPlatformSpecificConfigPath(platformConfig)
+
         def buildArgs = getBuildArgs(platformConfig, variant)
 
         if (additionalBuildArgs) {
@@ -98,6 +100,7 @@ class Builder implements Serializable {
                 DOCKER_IMAGE: dockerImage,
                 DOCKER_FILE: dockerFile,
                 DOCKER_NODE: dockerNode,
+                PLATFORM_CONFIG_PATH: platformSpecificConfigPath,
                 CONFIGURE_ARGS: getConfigureArgs(platformConfig, additionalConfigureArgs, variant),
                 OVERRIDE_FILE_NAME_VERSION: overrideFileNameVersion,
                 ADDITIONAL_FILE_NAME_TAG: platformConfig.additionalFileNameTag as String,
@@ -227,6 +230,14 @@ class Builder implements Serializable {
         return dockerNodeValue
     }
 
+    def getPlatformSpecificConfigPath(Map<String, ?> configuration) {
+        def platformSpecificConfigPath = ""
+        if (configuration.containsKey("platformSpecificConfigPath")) {
+            platformSpecificConfigPath = configuration.platformSpecificConfigPath
+        }
+        return platformSpecificConfigPath
+    }
+
     /**
      * Builds up a node param string that defines what nodes are eligible to run the given job
      * @param configuration

pipelines/build/common/config_regeneration.groovy

@@ -34,6 +34,8 @@ class Regeneration implements Serializable {
     private final def gitUri
     private final def gitBranch
 
+    private final def jobTemplatePath
+    private final def scriptPath
     private final def jenkinsBuildRoot
     private final def jenkinsUsername
     private final def jenkinsToken
@@ -53,6 +55,8 @@ class Regeneration implements Serializable {
         String jobRootDir,
         String gitUri,
         String gitBranch,
+        String jobTemplatePath,
+        String scriptPath,
         String jenkinsBuildRoot,
         String jenkinsUsername,
         String jenkinsToken
@@ -66,6 +70,8 @@ class Regeneration implements Serializable {
         this.jobRootDir = jobRootDir
         this.gitUri = gitUri
         this.gitBranch = gitBranch
+        this.jobTemplatePath = jobTemplatePath
+        this.scriptPath = scriptPath
         this.jenkinsBuildRoot = jenkinsBuildRoot
         this.jenkinsUsername = jenkinsUsername
         this.jenkinsToken = jenkinsToken
@@ -130,6 +136,14 @@ class Regeneration implements Serializable {
         return dockerNodeValue
     }
 
+    def getPlatformSpecificConfigPath(Map<String, ?> configuration) {
+        def platformSpecificConfigPath = ""
+        if (configuration.containsKey("platformSpecificConfigPath")) {
+            platformSpecificConfigPath = configuration.platformSpecificConfigPath
+        }
+        return platformSpecificConfigPath
+    }
+
     /**
     * Builds up a node param string that defines what nodes are eligible to run the given job. Used as a placeholder since the pipelines overwrite this.
     * @param configuration
@@ -253,6 +267,8 @@ class Regeneration implements Serializable {
 
             def dockerNode = getDockerNode(platformConfig, variant)
 
+            def platformSpecificConfigPath = getPlatformSpecificConfigPath(platformConfig)
+
             def buildArgs = getBuildArgs(platformConfig, variant)
 
             def testList = getTestList(platformConfig)
@@ -270,6 +286,7 @@ class Regeneration implements Serializable {
                 DOCKER_IMAGE: dockerImage,
                 DOCKER_FILE: dockerFile,
                 DOCKER_NODE: dockerNode,
+                PLATFORM_CONFIG_PATH: platformSpecificConfigPath,
                 CONFIGURE_ARGS: getConfigureArgs(platformConfig, variant),
                 OVERRIDE_FILE_NAME_VERSION: "",
                 ADDITIONAL_FILE_NAME_TAG: platformConfig.additionalFileNameTag as String,
@@ -306,13 +323,14 @@ class Regeneration implements Serializable {
         Map<String, ?> params = config.toMap().clone() as Map
         params.put("JOB_NAME", jobName)
         params.put("JOB_FOLDER", jobFolder)
+        params.put("SCRIPT_PATH", scriptPath)
 
         params.put("GIT_URI", gitUri)
         params.put("GIT_BRANCH", gitBranch)
 
         params.put("BUILD_CONFIG", config.toJson())
 
-        def create = context.jobDsl targets: "pipelines/build/common/create_job_from_template.groovy", ignoreExisting: false, additionalParameters: params
+        def create = context.jobDsl targets: jobTemplatePath, ignoreExisting: false, additionalParameters: params
 
         return create
     }
@@ -519,13 +537,17 @@ return {
     String jobRootDir,
     String gitUri,
     String gitBranch,
+    String jobTemplatePath,
+    String scriptPath,
     String jenkinsBuildRoot,
     String jenkinsUsername,
     String jenkinsToken
         ->
         if (jobRootDir == null) jobRootDir = "build-scripts";
         if (gitUri == null) gitUri = "https://github.com/AdoptOpenJDK/openjdk-build.git";
         if (gitBranch == null) gitBranch = "master";
+        if (jobTemplatePath == null) jobTemplatePath = "pipelines/build/common/create_job_from_template.groovy";
+        if (scriptPath == null) scriptPath = "pipelines/build/common/kick_off_build.groovy"
         if (jenkinsBuildRoot == null) jenkinsBuildRoot = "https://ci.adoptopenjdk.net/job/build-scripts/";
         if (jenkinsUsername == null) jenkinsUsername = ""
         if (jenkinsToken == null) jenkinsToken = ""
@@ -545,6 +567,8 @@ return {
             jobRootDir,
             gitUri,
             gitBranch,
+            jobTemplatePath,
+            scriptPath,
             jenkinsBuildRoot,
             jenkinsUsername,
             jenkinsToken

pipelines/build/common/create_job_from_template.groovy

@@ -45,7 +45,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
                     }
                 }
             }
-            scriptPath('pipelines/build/common/kick_off_build.groovy')
+            scriptPath("${SCRIPT_PATH}")
         }
     }
     properties {
@@ -74,6 +74,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
                 <dt><strong>CODEBUILD</strong></dt><dd>Use a dynamic codebuild machine if no other machine is available</dd>
                 <dt><strong>DOCKER_IMAGE</strong></dt><dd>Use a docker build environment</dd>
                 <dt><strong>DOCKER_FILE</strong></dt><dd>Relative path to a dockerfile to be built and used on top of the DOCKER_IMAGE</dd>
+                <dt><strong>PLATFORM_CONFIG_PATH</strong></dt><dd>Relative path to the platform specific configuration for this paticular OS</dd>
                 <dt><strong>CONFIGURE_ARGS</strong></dt><dd>Arguments for ./configure</dd>
                 <dt><strong>OVERRIDE_FILE_NAME_VERSION</strong></dt><dd>Set the version string on the file name</dd>
                 <dt><strong>RELEASE</strong></dt><dd>Is this build a release</dd>

pipelines/build/openjdk10_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,

pipelines/build/openjdk11_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,

pipelines/build/openjdk12_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,

pipelines/build/openjdk13_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,

pipelines/build/openjdk14_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,

pipelines/build/openjdk15_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,
@@ -49,4 +62,4 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
 } else {
     println "[ERROR] One or more setup parameters are null.\nscmVars = ${scmVars}\nconfigureBuild = ${configureBuild}\nbuildConfigurations = ${buildConfigurations}"
     throw new Exception()
-}
+}

pipelines/build/openjdk16_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/jdk16_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/jdk16_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,

pipelines/build/openjdk8_pipeline.groovy

@@ -20,11 +20,24 @@ def buildConfigurations = null
 node ("master") {
     scmVars = checkout scm
     load "${WORKSPACE}/pipelines/build/common/import_lib.groovy"
-    configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
-    buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+
+    // Load baseFilePath. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services.
+    if ("$baseFilePath" != "") {
+        configureBuild = load "${WORKSPACE}/${baseFilePath}"
+    } else {
+        configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy"
+    }
+
+    // Load buildConfigFilePath. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant.
+    if ("$buildConfigFilePath" != "") {
+        buildConfigurations = load "${WORKSPACE}/${buildConfigFilePath}"
+    } else {
+        buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy"
+    }
+
 }
 
-if (scmVars != null && configureBuild != null && buildConfigurations != null) {
+if (scmVars != null && (configureBuild != null || buildConfigurations != null)) {
     configureBuild(
         javaToBuild,
         buildConfigurations,