From c9e334deab4ebc346fb3ef5d729af36e4b1a28d1 Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Fri, 19 Mar 2021 12:34:02 -0400 Subject: [PATCH] Update condition Signed-off-by: Sophia Guo --- .gitignore | 54 ++++++++++++++-------------- buildenv/docker/Dockerfile | 61 ++++++++++++++++++++++++++++++-- buildenv/jenkins/JenkinsfileBase | 1 + buildenv/jenkins/openjdk_tests | 33 +++++++++++------ 4 files changed, 108 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 84c436b161..297d032bb8 100755 --- a/.gitignore +++ b/.gitignore @@ -1,27 +1,27 @@ -# java .class files -*.class - -# misc auto generated folders -bin -!perf/liberty/scripts/bin - -# misc auto generated files -autoGen.mk - -TestConfig -TKG -functional -!functional/security -!functional/SyntheticGCWorkload -!functional/MBCS_Tests - -Utils -systemtest_prereqs/* -openjdk/*.gz -.DS_Store -.idea/ - -/.classpath -/.project -/job.properties -.vscode/ +# java .class files +*.class + +# misc auto generated folders +bin +!perf/liberty/scripts/bin + +# misc auto generated files +autoGen.mk + +TestConfig +TKG +functional +!functional/security +!functional/SyntheticGCWorkload +!functional/MBCS_Tests + +Utils +systemtest_prereqs/* +openjdk/*.gz +.DS_Store +.idea/ + +/.classpath +/.project +/job.properties +.vscode/ diff --git a/buildenv/docker/Dockerfile b/buildenv/docker/Dockerfile index 7ba031d77f..b05414692a 100644 --- a/buildenv/docker/Dockerfile +++ b/buildenv/docker/Dockerfile @@ -15,9 +15,64 @@ # This Dockerfile in buildenv/docker dir is used to create an image with # AdoptOpenJDK jdk binary installed. Basic test dependent executions # are installed during the building process. +# +# Build example: `docker build -t adoptopenjdk-test .` +# +# This Dockerfile builds image based on adoptopenjdk/openjdk8:latest. +# If you want to build image based on other images, please use +# `--build-arg list` to specify your base image +# +# Build example: `docker build --build-arg IMAGE_NAME= --build-arg IMAGE_VERSION=-t adoptopenjdk-test .` + +ARG IMAGE_NAME=adoptopenjdk/openjdk8 +ARG IMAGE_VERSION=nightly + +FROM ${IMAGE_NAME}:${IMAGE_VERSION} + +# Use this when there is a need for input during docker image building process +ENV DEBIAN_FRONTEND noninteractive + +# Install required OS and testing tools +RUN apt-get update \ + && apt-get install -qq -y --no-install-recommends \ + ant \ + ant-contrib \ + autoconf \ + bash \ + build-essential \ + ca-certificates \ + git \ + make \ + perl \ + unzip \ + wget \ + vim \ + zip \ + xserver-xorg-core \ + xserver-xorg \ + xauth \ + xorg \ + openbox \ + x11vnc \ + xvfb \ + ftpd \ + krb5-kdc \ + krb5-admin-server \ + pwgen \ + tomcat8 \ + libxml-parser-perl \ + && rm -rf /var/lib/apt/lists/* + +# Install Docker module to run test framework +RUN echo yes | cpan install JSON Text::CSV -ARG IMAGE=adoptopenjdk/centos6_build_image +# These two folders are used testing purpose. +# In the `docker run` command, user also need mount +# and to the Docker image by using cmd: +# `docker run -it -v :/java -v :/test ` +VOLUME ["/test","/java"] -FROM $image +# Shell script to automate test execution +COPY ./mktest.sh /mktest.sh -//specific prereqs \ No newline at end of file +CMD /bin/bash /mktest.sh \ No newline at end of file diff --git a/buildenv/jenkins/JenkinsfileBase b/buildenv/jenkins/JenkinsfileBase index 4af874f15f..810d4d4c45 100644 --- a/buildenv/jenkins/JenkinsfileBase +++ b/buildenv/jenkins/JenkinsfileBase @@ -66,6 +66,7 @@ def setupEnv() { SSH_AGENT_CREDENTIAL = params.SSH_AGENT_CREDENTIAL ? params.SSH_AGENT_CREDENTIAL : "" KEEP_WORKSPACE = params.KEEP_WORKSPACE ? params.KEEP_WORKSPACE : false OPENJ9_SHA = params.OPENJ9_SHA ? params.OPENJ9_SHA : "" + CLOUD_PROVIDER = params.CLOUD_PROVIDER ? params.CLOUD_PROVIDER : "" env.JDK_BRANCH = params.JDK_BRANCH ? params.JDK_BRANCH : "" env.USER_CREDENTIALS_ID = params.USER_CREDENTIALS_ID ? params.USER_CREDENTIALS_ID : "" env.TEST_JDK_HOME = "$WORKSPACE/openjdkbinary/j2sdk-image" diff --git a/buildenv/jenkins/openjdk_tests b/buildenv/jenkins/openjdk_tests index 8d36dad0e9..3ebf02850e 100644 --- a/buildenv/jenkins/openjdk_tests +++ b/buildenv/jenkins/openjdk_tests @@ -68,18 +68,22 @@ def PLATFORM_MAP = [ 's390_linux' : [ 'SPEC' : 'linux_390', 'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.linux&&hw.bits.32', + 'DynamicAgents' : ['fyre'] ], 's390x_linux' : [ 'SPEC' : 'linux_390-64_cmprssptrs', 'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.linux', + 'DynamicAgents' : ['fyre'] ], 's390x_linux_mixed' : [ 'SPEC' : 'linux_390-64_mxdptrs', 'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.linux', + 'DynamicAgents' : ['fyre'] ], 's390x_linux_xl' : [ 'SPEC' : 'linux_390-64', 'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.linux', + 'DynamicAgents' : ['fyre'] ], 's390x_zos' : [ 'SPEC' : 'zos_390-64_cmprssptrs', @@ -115,15 +119,18 @@ def PLATFORM_MAP = [ ], 'x86-64_linux' : [ 'SPEC' : 'linux_x86-64_cmprssptrs', - 'LABEL' : 'test-ibmcloud-ubuntu1604-x64-1', + 'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.linux', + 'DynamicAgents' : ['azure', 'fyre'] ], 'x86-64_linux_mixed' : [ 'SPEC' : 'linux_x86-64_mxdptrs', 'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.linux', + 'DynamicAgents' : ['azure', 'fyre'] ], 'x86-64_linux_xl' : [ 'SPEC' : 'linux_x86-64', 'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.linux', + 'DynamicAgents' : ['azure', 'fyre'] ], 'x86-64_mac' : [ 'SPEC' : 'osx_x86-64_cmprssptrs', @@ -144,17 +151,18 @@ def PLATFORM_MAP = [ 'x86-64_windows_mixed' : [ 'SPEC' : 'win_x86-64_mxdptrs', 'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.windows', + 'DynamicAgents' : ['fyre'] ], 'x86-64_windows_xl' : [ 'SPEC' : 'win_x86-64', 'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.windows', + 'DynamicAgents' : ['fyre'] ], ] if ( !params.TARGET ) { assert false : "Please provide TARGET value" } -def X64Dockeimage = 'adoptopenjdk/centos6_build_image' def PLATFORMS = params.PLATFORM.trim().split("\\s*,\\s*"); def JDK_VERSIONS = params.JDK_VERSION.trim().split("\\s*,\\s*"); def JDK_IMPLS = params.JDK_IMPL.trim().split("\\s*,\\s*"); @@ -210,6 +218,10 @@ if (JDK_VERSIONS.size() > 1 || JDK_IMPLS.size() > 1 || PLATFORMS.size() >1 || PL if (PLATFORM_MAP.containsKey(params.PLATFORM)) { SPEC = PLATFORM_MAP[params.PLATFORM]["SPEC"] LABEL = params.LABEL ? params.LABEL : PLATFORM_MAP[params.PLATFORM]["LABEL"] + dynamicAgents = [] + if (PLATFORM_MAP[params.PLATFORM].containsKey('DynamicAgents')) { + dynamicAgents = PLATFORM_MAP[params.PLATFORM]["DynamicAgents"] + } if (params.DOCKER_REQUIRED) { LABEL += "&&sw.tool.docker" @@ -236,7 +248,7 @@ if (JDK_VERSIONS.size() > 1 || JDK_IMPLS.size() > 1 || PLATFORMS.size() >1 || PL boolean isNodeIdle = false node('master') { for (String onlineNode : onlineNodes) { - def nodeStatus = sh(script: "curl ${${env.JENKINS_URL}}computer/${onlineNode}/api/xml?xpath=/*/idle", + def nodeStatus = sh(script: "curl ${env.JENKINS_URL}computer/${onlineNode}/api/xml?xpath=/*/idle", returnStdout: true, returnStatus: false ).trim() @@ -248,12 +260,11 @@ if (JDK_VERSIONS.size() > 1 || JDK_IMPLS.size() > 1 || PLATFORMS.size() >1 || PL } } // IF no nodes are idle we will check if there is supported virtual agent - // This condition should be updated according to updated supported virtual agent. - // For CLOUD_PRIVIDER is FRYE, we need to update what is the fyre supported platform and update it's label + // When Parallel the race condition could happen. Say the number of multiply jobs is larger than the available nodes the query's result may be delayed and wrong + // In this case jobs will be fooled to fall back to wait local busy nodes. if (!isNodeIdle && (CLOUD_PROVIDER.length()) != 0) { - if (LABEL.contains('test-ibmcloud-ubuntu1604-x64-1') - || ((CLOUD_PROVIDER.contains('frye') && LABEL.contains('ci.role.test&&hw.arch.x86&&sw.os.linux')))) { - LABEL = 'ci.agent.dynamic' + if (CLOUD_PROVIDER in dynamicAgents) { + LABEL += 'ci.agent.dynamic' } } @@ -290,9 +301,9 @@ if (JDK_VERSIONS.size() > 1 || JDK_IMPLS.size() > 1 || PLATFORMS.size() >1 || PL ] } jenkinsfile = load "${WORKSPACE}/openjdk-tests/buildenv/jenkins/JenkinsfileBase" - if (LABEL.contains('ci.agent.dynamic')) { - //adoptopenjdk/centos6_build_image works for adoptopenjdk jenkins. For openj9 ? - docker.image('adoptopenjdk/centos6_build_image').pull() + if (LABEL.contains('ci.agent.dynamic') && CLOUD_PROVIDER == 'azure') { + //Set dockerimage for azure agent. Fyre has stencil to setup the right environment + docker.image('adoptopenjdk/centos6_build_image').pull() docker.image('adoptopenjdk/centos6_build_image').inside { jenkinsfile.testBuild() }