diff --git a/build-farm/set-platform-specific-configurations.sh b/build-farm/set-platform-specific-configurations.sh index dfcfb8d01a..93321cba6f 100755 --- a/build-farm/set-platform-specific-configurations.sh +++ b/build-farm/set-platform-specific-configurations.sh @@ -26,4 +26,9 @@ fi export VARIANT_ARG="--build-variant ${VARIANT}" # shellcheck disable=SC1091,SC1090 -source "$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh" \ No newline at end of file +if [ ! -z "$PLATFORM_CONFIG_PATH" ] +then + source "$SCRIPT_DIR/${PLATFORM_CONFIG_PATH}" +else + source "$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh" +fi diff --git a/pipelines/build/common/build_base_file.groovy b/pipelines/build/common/build_base_file.groovy index a95eb5aa28..4a36c7709c 100644 --- a/pipelines/build/common/build_base_file.groovy +++ b/pipelines/build/common/build_base_file.groovy @@ -71,6 +71,8 @@ class Builder implements Serializable { def dockerNode = getDockerNode(platformConfig, variant) + def platformSpecificConfigPath = getPlatformSpecificConfigPath(platformConfig) + def buildArgs = getBuildArgs(platformConfig, variant) if (additionalBuildArgs) { @@ -98,6 +100,7 @@ class Builder implements Serializable { DOCKER_IMAGE: dockerImage, DOCKER_FILE: dockerFile, DOCKER_NODE: dockerNode, + PLATFORM_CONFIG_PATH: platformSpecificConfigPath, CONFIGURE_ARGS: getConfigureArgs(platformConfig, additionalConfigureArgs, variant), OVERRIDE_FILE_NAME_VERSION: overrideFileNameVersion, ADDITIONAL_FILE_NAME_TAG: platformConfig.additionalFileNameTag as String, @@ -227,6 +230,14 @@ class Builder implements Serializable { return dockerNodeValue } + def getPlatformSpecificConfigPath(Map configuration) { + def platformSpecificConfigPath = "" + if (configuration.containsKey("platformSpecificConfigPath")) { + platformSpecificConfigPath = configuration.platformSpecificConfigPath + } + return platformSpecificConfigPath + } + /** * Builds up a node param string that defines what nodes are eligible to run the given job * @param configuration diff --git a/pipelines/build/common/config_regeneration.groovy b/pipelines/build/common/config_regeneration.groovy index 93cebed984..6b16af3af7 100644 --- a/pipelines/build/common/config_regeneration.groovy +++ b/pipelines/build/common/config_regeneration.groovy @@ -130,6 +130,14 @@ class Regeneration implements Serializable { return dockerNodeValue } + def getPlatformSpecificConfigPath(Map configuration) { + def platformSpecificConfigPath = "" + if (configuration.containsKey("platformSpecificConfigPath")) { + platformSpecificConfigPath = configuration.platformSpecificConfigPath + } + return platformSpecificConfigPath + } + /** * Builds up a node param string that defines what nodes are eligible to run the given job. Used as a placeholder since the pipelines overwrite this. * @param configuration @@ -253,6 +261,8 @@ class Regeneration implements Serializable { def dockerNode = getDockerNode(platformConfig, variant) + def platformSpecificConfigPath = getPlatformSpecificConfigPath(platformConfig) + def buildArgs = getBuildArgs(platformConfig, variant) def testList = getTestList(platformConfig) @@ -270,6 +280,7 @@ class Regeneration implements Serializable { DOCKER_IMAGE: dockerImage, DOCKER_FILE: dockerFile, DOCKER_NODE: dockerNode, + PLATFORM_CONFIG_PATH: platformSpecificConfigPath, CONFIGURE_ARGS: getConfigureArgs(platformConfig, variant), OVERRIDE_FILE_NAME_VERSION: "", ADDITIONAL_FILE_NAME_TAG: platformConfig.additionalFileNameTag as String, diff --git a/pipelines/build/common/create_job_from_template.groovy b/pipelines/build/common/create_job_from_template.groovy index ca6513c997..0f6751ea0d 100644 --- a/pipelines/build/common/create_job_from_template.groovy +++ b/pipelines/build/common/create_job_from_template.groovy @@ -74,6 +74,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
CODEBUILD
Use a dynamic codebuild machine if no other machine is available
DOCKER_IMAGE
Use a docker build environment
DOCKER_FILE
Relative path to a dockerfile to be built and used on top of the DOCKER_IMAGE
+
PLATFORM_CONFIG_PATH
Relative path to the platform specific configuration for this paticular OS
CONFIGURE_ARGS
Arguments for ./configure
OVERRIDE_FILE_NAME_VERSION
Set the version string on the file name
RELEASE
Is this build a release
diff --git a/pipelines/build/openjdk8_pipeline.groovy b/pipelines/build/openjdk8_pipeline.groovy index f32020784f..afd95f7e0f 100644 --- a/pipelines/build/openjdk8_pipeline.groovy +++ b/pipelines/build/openjdk8_pipeline.groovy @@ -20,11 +20,25 @@ def buildConfigurations = null node ("master") { scmVars = checkout scm load "${WORKSPACE}/pipelines/build/common/import_lib.groovy" - configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy" - buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy" + + // Load BASE_FILE_PATH. This is where build_base_file.groovy is located. It runs the downstream job setup and configuration retrieval services. + def configureBuild = null + if ("$BASE_FILE_PATH" != "") { + configureBuild = load "${WORKSPACE}/${BASE_FILE_PATH}" + } else { + configureBuild = load "${WORKSPACE}/pipelines/build/common/build_base_file.groovy" + } + + // Load BUILD_CONFIG_FILE_PATH. This is where jdkxx_pipeline_config.groovy is located. It contains the build configurations for each platform, architecture and variant. + if ("$BUILD_CONFIG_FILE_PATH" != "") { + buildConfigurations = load "${WORKSPACE}/${BUILD_CONFIG_FILE_PATH}" + } else { + buildConfigurations = load "${WORKSPACE}/pipelines/jobs/configurations/${javaToBuild}_pipeline_config.groovy" + } + } -if (scmVars != null && configureBuild != null && buildConfigurations != null) { +if (scmVars != null && (configureBuild != null || buildConfigurations != null)) { configureBuild( javaToBuild, buildConfigurations, diff --git a/pipelines/build/regeneration/build_pipeline_generator.groovy b/pipelines/build/regeneration/build_pipeline_generator.groovy index 5ed17bc2a4..79352eee97 100644 --- a/pipelines/build/regeneration/build_pipeline_generator.groovy +++ b/pipelines/build/regeneration/build_pipeline_generator.groovy @@ -1,5 +1,33 @@ +import java.nio.file.NoSuchFileException + node('master') { - def retiredVersions = [9, 10, 12, 13, 14] + def retiredVersions = [9, 10, 12, 13] + def generatedPipelines = [] + + // Load gitUri and gitBranch. These determine where we will be pulling configs from. + def repoUri = "$REPOSITORY_URL" != "" ? REPOSITORY_URL : "https://github.com/AdoptOpenJDK/openjdk-build.git" + def repoBranch = "$REPOSITORY_BRANCH" != "" ? REPOSITORY_BRANCH : "master" + + // Load jobRoot. This is where the openjdkxx-pipeline jobs will be created. + def jobRoot = "$JOB_ROOT" != "" ? JOB_ROOT : "build-scripts" + + // Load scriptFolderPath. This is the folder where the openjdkxx-pipeline.groovy code is located compared to the repository root. These are the top level pipeline jobs. + def scriptFolderPath = "$SCRIPT_FOLDER_PATH" != "" ? SCRIPT_FOLDER_PATH : "pipelines/build" + + // Load nightlyFolderPath. This is the folder where the jdkxx.groovy code is located compared to the repository root. These define what the default set of nightlies will be. + def nightlyFolderPath = "$NIGHTLY_FOLDER_PATH" != "" ? NIGHTLY_FOLDER_PATH : "pipelines/jobs/configurations" + + // Load jobTemplatePath. This is where the pipeline_job_template.groovy code is located compared to the repository root. This actually sets up the pipeline job using the parameters above. + def jobTemplatePath = "$JOB_TEMPLATE_PATH" != "" ? JOB_TEMPLATE_PATH : "pipelines/jobs/pipeline_job_template.groovy" + + println "[INFO] Running generator script with the following configuration:" + println "REPOSITORY_URL = $repoUri" + println "REPOSITORY_BRANCH = $repoBranch" + println "JOB_ROOT = $jobRoot" + println "SCRIPT_FOLDER_PATH = $scriptFolderPath" + println "NIGHTLY_FOLDER_PATH = $nightlyFolderPath" + println "JOB_TEMPLATE_PATH = $jobTemplatePath" + println "ENABLE_PIPELINE_SCHEDULE = $ENABLE_PIPELINE_SCHEDULE" (8..30).each({javaVersion -> @@ -7,26 +35,33 @@ node('master') { println "[INFO] $javaVersion is a retired version that isn't built anymore. Skipping generation..." return } - + def config = [ TEST : false, - GIT_URL : "https://github.com/AdoptOpenJDK/openjdk-build.git", - BRANCH : "master", - BUILD_FOLDER : "build-scripts", + GIT_URL : repoUri, + BRANCH : repoBranch, + BUILD_FOLDER : jobRoot, JOB_NAME : "openjdk${javaVersion}-pipeline", - SCRIPT : "pipelines/build/openjdk${javaVersion}_pipeline.groovy", + SCRIPT : "${scriptFolderPath}/openjdk${javaVersion}_pipeline.groovy", disableJob : false, triggerSchedule : "" ]; - checkout([$class: 'GitSCM', userRemoteConfigs: [[url: config.GIT_URL]]]) + + checkout( + [ + $class: 'GitSCM', + branches: [[name: config.BRANCH ]], + userRemoteConfigs: [[ url: config.GIT_URL ]] + ] + ) def target; try { - target = load "${WORKSPACE}/pipelines/jobs/configurations/jdk${javaVersion}u.groovy" - } catch(Exception e) { + target = load "${WORKSPACE}/${nightlyFolderPath}/jdk${javaVersion}u.groovy" + } catch(NoSuchFileException e) { try { - target = load "${WORKSPACE}/pipelines/jobs/configurations/jdk${javaVersion}.groovy" - } catch(Exception e2) { + target = load "${WORKSPACE}/${nightlyFolderPath}/jdk${javaVersion}.groovy" + } catch(NoSuchFileException e2) { println "[WARNING] No config found for JDK${javaVersion}" return } @@ -43,7 +78,8 @@ node('master') { println "[INFO] JDK${javaVersion}: disableJob = ${config.disableJob}" - if (Boolean.parseBoolean(enablePipelineSchedule) == true) { + // Load ENABLE_PIPELINE_SCHEDULE. This determines whether the jobs should run automatically or not. + if (Boolean.parseBoolean(ENABLE_PIPELINE_SCHEDULE) == true) { try { config.triggerSchedule = target.triggerSchedule } catch (Exception ex) { @@ -53,7 +89,18 @@ node('master') { println "[INFO] JDK${javaVersion}: triggerSchedule = ${config.triggerSchedule}" - def create = jobDsl targets: "pipelines/jobs/pipeline_job_template.groovy", ignoreExisting: false, additionalParameters: config + def create = jobDsl targets: jobTemplatePath, ignoreExisting: false, additionalParameters: config target.disableJob = false + + generatedPipelines.add(javaVersion) }) + + // Fail if nothing was generated + if (generatedPipelines == []) { + throw new Exception("[ERROR] NO PIPELINES WERE GENERATED!") + } else { + println "[SUCCESS] THE FOLLOWING PIPELINES WERE GENERATED IN THE $JOB_ROOT FOLDER" + println generatedPipelines + } + } diff --git a/pipelines/build/regeneration/jdk8_regeneration_pipeline.groovy b/pipelines/build/regeneration/jdk8_regeneration_pipeline.groovy index 970bf8eaac..ccfad62803 100644 --- a/pipelines/build/regeneration/jdk8_regeneration_pipeline.groovy +++ b/pipelines/build/regeneration/jdk8_regeneration_pipeline.groovy @@ -21,6 +21,10 @@ node ("master") { def scmVars = checkout scm load "${WORKSPACE}/pipelines/build/common/import_lib.groovy" + // Load gitUri and gitBranch. These determine where we will be pulling configs from. + def repoUri = "$REPOSITORY_URL" != "" ? REPOSITORY_URL : null + def repoBranch = "$REPOSITORY_BRANCH" != "" ? REPOSITORY_BRANCH : null + // Load buildConfigurations from config file. This is what the nightlies & releases use to setup their downstream jobs def buildConfigurations = null def buildConfigPath = "${WORKSPACE}/pipelines/jobs/configurations/${javaVersion}_pipeline_config.groovy" @@ -64,6 +68,8 @@ node ("master") { println "[INFO] Running regeneration script with the following configuration:" println "VERSION: $javaVersion" + println "REPOSITORY URL: $repoUri" + println "REPOSITORY BRANCH: $repoBranch" println "BUILD CONFIGURATIONS: $buildConfigurations" println "JOBS TO GENERATE: $targetConfigurations" println "JOB ROOT: $jobRoot" @@ -91,8 +97,8 @@ node ("master") { currentBuild, this, jobRoot, - null, - null, + repoUri, + repoBranch, jenkinsBuildRoot, jenkinsUsername, jenkinsToken @@ -110,8 +116,8 @@ node ("master") { currentBuild, this, jobRoot, - null, - null, + repoUri, + repoBranch, jenkinsBuildRoot, null, null diff --git a/pipelines/library/src/common/IndividualBuildConfig.groovy b/pipelines/library/src/common/IndividualBuildConfig.groovy index 3fc172ab90..8cb0395a3c 100644 --- a/pipelines/library/src/common/IndividualBuildConfig.groovy +++ b/pipelines/library/src/common/IndividualBuildConfig.groovy @@ -16,6 +16,7 @@ class IndividualBuildConfig implements Serializable { final String DOCKER_IMAGE final String DOCKER_FILE final String DOCKER_NODE + final String PLATFORM_CONFIG_PATH final String CONFIGURE_ARGS final String OVERRIDE_FILE_NAME_VERSION final String ADDITIONAL_FILE_NAME_TAG @@ -52,6 +53,7 @@ class IndividualBuildConfig implements Serializable { DOCKER_IMAGE = map.get("DOCKER_IMAGE") DOCKER_FILE = map.get("DOCKER_FILE") DOCKER_NODE = map.get("DOCKER_NODE") + PLATFORM_CONFIG_PATH = map.get("PLATFORM_CONFIG_PATH") CONFIGURE_ARGS = map.get("CONFIGURE_ARGS") OVERRIDE_FILE_NAME_VERSION = map.get("OVERRIDE_FILE_NAME_VERSION") ADDITIONAL_FILE_NAME_TAG = map.get("ADDITIONAL_FILE_NAME_TAG") @@ -93,6 +95,7 @@ class IndividualBuildConfig implements Serializable { DOCKER_IMAGE : DOCKER_IMAGE, DOCKER_FILE : DOCKER_FILE, DOCKER_NODE : DOCKER_NODE, + PLATFORM_CONFIG_PATH : PLATFORM_CONFIG_PATH, CONFIGURE_ARGS : CONFIGURE_ARGS, OVERRIDE_FILE_NAME_VERSION: OVERRIDE_FILE_NAME_VERSION, ADDITIONAL_FILE_NAME_TAG : ADDITIONAL_FILE_NAME_TAG,