diff --git a/.github/workflows/groovy-tests.yml b/.github/workflows/groovy-tests.yml new file mode 100644 index 000000000..48ef54f83 --- /dev/null +++ b/.github/workflows/groovy-tests.yml @@ -0,0 +1,16 @@ +name: groovy-tests + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: 11 + - uses: actions/checkout@v2 + - name: Run Tests + run: | + ./gradlew test --info diff --git a/.github/workflows/tests.yml b/.github/workflows/python-tests.yml similarity index 98% rename from .github/workflows/tests.yml rename to .github/workflows/python-tests.yml index e3d54cc72..b22a0d22a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/python-tests.yml @@ -1,4 +1,4 @@ -name: tests +name: python-tests on: [push, pull_request] diff --git a/.gitignore b/.gitignore index 4cd76cfec..cc653f2d6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,10 +12,12 @@ coverage.xml .gradle/ .settings/ +/build/ out.txt /builds/ /dist/ /test-results/ -/.vscode/ \ No newline at end of file +/.vscode/ + diff --git a/.groovylintrc.json b/.groovylintrc.json new file mode 100644 index 000000000..481a6cb74 --- /dev/null +++ b/.groovylintrc.json @@ -0,0 +1,8 @@ +{ + "extends": "recommended", + "rules": { + "CompileStatic": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 3d205600e..cc34c391f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -6,6 +6,7 @@ - [Pipenv](#pipenv) - [NVM and Node](#nvm-and-node) - [Yarn](#yarn) + - [Java](#java) - [Install Dependencies](#install-dependencies) - [Run Tests](#run-tests) - [Build OpenSearch](#build-opensearch) @@ -76,6 +77,10 @@ nvm install v10.24.1 npm install -g yarn ``` +#### Java + +This project recommends Java 11 for Jenkins jobs CI. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). + ### Install Dependencies Install dependencies. @@ -90,13 +95,21 @@ Alternatively, run a command inside the virtualenv with pipenv run. ### Run Tests -This project uses [pytest](https://docs.pytest.org/en/6.x/) to ensure code quality. See [tests](tests). +This project uses [pytest](https://docs.pytest.org/en/6.x/) to ensure Python code quality, and [JUnit](https://junit.org/) for Groovy code. See [tests](tests). ``` $ pipenv run pytest 2 passed in 02s ``` +``` +$ ./gradlew test + +> Task :test +BUILD SUCCESSFUL in 7s +3 actionable tasks: 1 executed, 2 up-to-date +``` + ### Build OpenSearch Try running `./build.sh`. It should complete and show usage. @@ -115,7 +128,7 @@ build.sh: error: the following arguments are required: manifest ### Code Linting -This project uses a [pre-commit hook](https://pre-commit.com/) for linting python code. +This project uses a [pre-commit hook](https://pre-commit.com/) for linting Python code. ``` $ pipenv run pre-commit install diff --git a/build.gradle b/build.gradle new file mode 100644 index 000000000..af338fe01 --- /dev/null +++ b/build.gradle @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * This build file was generated by the Gradle 'init' task. + * + * This generated file contains a commented-out sample Java project to get you started. + * For more details take a look at the Java Quickstart chapter in the Gradle + * user guide available at https://docs.gradle.org/3.5/userguide/tutorial_java_projects.html + */ + +plugins { + id 'com.mkobit.jenkins.pipelines.shared-library' version '0.10.1' + id 'jacoco' + id 'java' + id 'groovy' +} + +repositories { + maven { url 'http://bits.netbeans.org/maven2/' } + maven { url 'https://repo.jenkins-ci.org/releases/' } + jcenter() + maven { url 'https://mvnrepository.com/artifact/' } + mavenLocal() +} + +dependencies { + compile group: 'org.assertj', name: 'assertj-core', version: '3.4.1' + compile group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.7' + compile group: 'com.cloudbees', name: 'groovy-cps', version: '1.12' +} + +sourceSets { + main { + groovy { + srcDirs = ['src/jenkins'] + } + } + + test { + groovy { + srcDirs = ['tests/jenkins'] + } + } + + jobs { + groovy { + srcDirs 'src/jenkins/jobs' + compileClasspath += main.compileClasspath + } + + compileClasspath += sourceSets.main.output + runtimeClasspath += sourceSets.main.output + } +} + +sharedLibrary { + coreVersion = '2.176.2' + testHarnessVersion = '2.54' + pluginDependencies { + workflowCpsGlobalLibraryPluginVersion = '2.16' + dependency('org.jenkins-ci.plugins', 'pipeline-input-step', '2.8') + } +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 000000000..e424d8de1 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,13 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +org.gradle.warning.mode=none +org.gradle.parallel=true diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..e708b1c02 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..d89a20d21 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,18 @@ + +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionSha256Sum=11657af6356b7587bfb37287b5992e94a9686d5c8a0a1b60b87b9928a2decde5 \ No newline at end of file diff --git a/gradlew b/gradlew new file mode 100755 index 000000000..3a163de71 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" \ No newline at end of file diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 000000000..28690fe08 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega \ No newline at end of file diff --git a/jenkins/opensearch-dashboards/Jenkinsfile b/jenkins/opensearch-dashboards/Jenkinsfile index 8e025ba5f..3d95a5001 100644 --- a/jenkins/opensearch-dashboards/Jenkinsfile +++ b/jenkins/opensearch-dashboards/Jenkinsfile @@ -1,3 +1,5 @@ +lib = library(identifier: "jenkins@current", retriever: legacySCM(scm)).jenkins + pipeline { agent none triggers { @@ -130,7 +132,6 @@ pipeline { post() { success { node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { - publishNotification(":white_check_mark:", "Successful Build", "\n${getAllJenkinsMessages()}") script { def VERSION = sh(script: 'echo ${INPUT_MANIFEST} | grep -Po "[0-9.]+?(?=.yml)"', returnStdout: true).trim() sh "echo VERSION:${VERSION} BUILD_NUMBER:${env.BUILD_NUMBER}" @@ -145,6 +146,9 @@ pipeline { booleanParam(name: 'IS_STAGING', value: true) ] } + + def stashed = lib.Messages.new(this).get(['build-x64', 'build-arm64']) + publishNotification(":white_check_mark:", "Successful Build", "\n${stashed}") } } } @@ -167,7 +171,7 @@ void build(platform, architecture) { void assemble() { git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' - script { manifest = readYaml(file: 'builds/manifest.yml') } + manifest = readYaml(file: 'builds/manifest.yml') def artifactPath = "${env.JOB_NAME}/${manifest.build.version}/${env.BUILD_NUMBER}/${manifest.build.platform}/${manifest.build.architecture}"; def BASE_URL = "${PUBLIC_ARTIFACT_URL}/${artifactPath}"; @@ -179,8 +183,7 @@ void assemble() { s3Upload(file: "dist", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${artifactPath}/dist") } - addJenkinsMessage("${BASE_URL}/builds/manifest.yml\n" + - "${BASE_URL}/dist/manifest.yml") + lib.Messages.new(this).add("${STAGE_NAME}", "${BASE_URL}/builds/manifest.yml\n${BASE_URL}/dist/manifest.yml") } /** Publishes a notification to a slack instance*/ @@ -189,33 +192,3 @@ void publishNotification(icon, msg, extra) { sh("""curl -XPOST --header "Content-Type: application/json" --data '{"result_text": "$icon ${env.JOB_NAME} [${env.BUILD_NUMBER}] $msg ${env.BUILD_URL}\nManifest: ${INPUT_MANIFEST} $extra"}' """ + "$TOKEN") } } - -/** Add a message to the jenkins queue */ -void addJenkinsMessage(message) { - writeFile(file: "notifications/${STAGE_NAME}.msg", text: message) - stash(includes: "notifications/*" , name: "notifications-${STAGE_NAME}") -} - -/** Load all message in the jenkins queue and append them with a leading newline into a mutli-line string */ -String getAllJenkinsMessages() { - script { - // Stages must be explicitly added to prevent overwriting - // See https://ryan.himmelwright.net/post/jenkins-parallel-stashing/ - def stages = ['build-linux-x64', 'post-build (linux-arm64)'] - for (stage in stages) { - unstash "notifications-${stage}" - } - - def files = findFiles(excludes: '', glob: 'notifications/*') - def data = "" - for (file in files) { - data = data + "\n" + readFile (file: file.path) - } - - // Delete all the notifications from the workspace - dir('notifications') { - deleteDir() - } - return data - } -} diff --git a/jenkins/opensearch/Jenkinsfile b/jenkins/opensearch/Jenkinsfile index d1f968054..6ee08cf8d 100644 --- a/jenkins/opensearch/Jenkinsfile +++ b/jenkins/opensearch/Jenkinsfile @@ -1,3 +1,5 @@ +lib = library(identifier: "jenkins@current", retriever: legacySCM(scm)).jenkins + pipeline { agent none triggers { @@ -73,7 +75,7 @@ pipeline { git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' sh "./build.sh manifests/$INPUT_MANIFEST --snapshot" withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { - sh('$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/builds/$ARTIFACT_PATH/maven') + sh('$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/builds/maven') } } } @@ -131,7 +133,6 @@ pipeline { post() { success { node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { - publishNotification(":white_check_mark:", "Successful Build", "\n${getAllJenkinsMessages()}") script { def VERSION = sh(script: 'echo ${INPUT_MANIFEST} | grep -Po "[0-9.]+?(?=.yml)"', returnStdout: true).trim() sh "echo VERSION:${VERSION} BUILD_NUMBER:${env.BUILD_NUMBER}" @@ -146,6 +147,9 @@ pipeline { booleanParam(name: 'IS_STAGING', value: true) ] } + + def stashed = lib.Messages.new(this).get(['build-x64', 'build-arm64']) + publishNotification(":white_check_mark:", "Successful Build", "\n${stashed}") } } } @@ -164,7 +168,7 @@ void build() { sh "./build.sh manifests/$INPUT_MANIFEST" - script { manifest = readYaml(file: 'builds/manifest.yml') } + manifest = readYaml(file: 'builds/manifest.yml') def artifactPath = "${env.JOB_NAME}/${manifest.build.version}/${env.BUILD_NUMBER}/${manifest.build.platform}/${manifest.build.architecture}"; def BASE_URL = "${PUBLIC_ARTIFACT_URL}/${artifactPath}"; @@ -176,9 +180,7 @@ void build() { s3Upload(file: "dist", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${artifactPath}/dist") } - addJenkinsMessage("${BASE_URL}/builds/manifest.yml\n" + - "${BASE_URL}/dist/manifest.yml") - + lib.Messages.new(this).add("${STAGE_NAME}", "${BASE_URL}/builds/manifest.yml\n${BASE_URL}/dist/manifest.yml") } /** Publishes a notification to a slack instance*/ @@ -186,34 +188,4 @@ void publishNotification(icon, msg, extra) { withCredentials([string(credentialsId: 'BUILD_NOTICE_WEBHOOK', variable: 'TOKEN')]) { sh("""curl -XPOST --header "Content-Type: application/json" --data '{"result_text": "$icon ${env.JOB_NAME} [${env.BUILD_NUMBER}] $msg ${env.BUILD_URL}\nManifest: ${INPUT_MANIFEST} $extra"}' """ + "$TOKEN") } -} - -/** Add a message to the jenkins queue */ -void addJenkinsMessage(message) { - writeFile(file: "notifications/${STAGE_NAME}.msg", text: message) - stash(includes: "notifications/*" , name: "notifications-${STAGE_NAME}") -} - -/** Load all message in the jenkins queue and append them with a leading newline into a mutli-line string */ -String getAllJenkinsMessages() { - script { - // Stages must be explicitly added to prevent overwriting - // See https://ryan.himmelwright.net/post/jenkins-parallel-stashing/ - def stages = ['build-x64', 'build-arm64'] - for (stage in stages) { - unstash "notifications-${stage}" - } - - def files = findFiles(excludes: '', glob: 'notifications/*') - def data = "" - for (file in files) { - data = data + "\n" + readFile (file: file.path) - } - - // Delete all the notifications from the workspace - dir('notifications') { - deleteDir() - } - return data - } -} +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 000000000..c4601f697 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,11 @@ +/* + * This settings file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * In a single project build this file can be empty or even removed. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user guide at https://docs.gradle.org/3.5/userguide/multi_project_builds.html + */ + +rootProject.name = 'jenkins-commons' \ No newline at end of file diff --git a/src/jenkins/Messages.groovy b/src/jenkins/Messages.groovy new file mode 100644 index 000000000..d07b8cf08 --- /dev/null +++ b/src/jenkins/Messages.groovy @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins + +class Messages implements Serializable { + def steps + + Messages(steps) { + this.steps = steps + } + + // Add a message to the Jenkins queue. + def add(String stage, String message) { + this.steps.writeFile(file: "messages/${stage}.msg", text: message) + this.steps.stash(includes: "messages/*" , name: "messages-${stage}") + } + + // Load all message in the jenkins queue and append them with a leading newline into a mutli-line string. + String get(ArrayList stages) { + // Stages must be explicitly added to prevent overwriting + // see https://ryan.himmelwright.net/post/jenkins-parallel-stashing/ + + for (stage in stages) { + this.steps.unstash(name: "messages-${stage}") + } + + def files = this.steps.findFiles(excludes: '', glob: 'messages/*') + def data = "" + for (file in files) { + data = data + "\n" + this.steps.readFile(file: file.path) + } + + this.steps.dir('messages') { + this.steps.deleteDir() + } + + return data + } +} \ No newline at end of file diff --git a/tests/jenkins/TestBuild.groovy b/tests/jenkins/TestBuild.groovy new file mode 100644 index 000000000..2bedc6acc --- /dev/null +++ b/tests/jenkins/TestBuild.groovy @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import org.junit.* +import com.lesfurets.jenkins.unit.BasePipelineTest +import com.lesfurets.jenkins.unit.MethodCall +import static org.assertj.core.api.Assertions.assertThat + +class TestBuild extends BasePipelineTest { + def jenkinsScript = "tests/jenkins/jobs/build.groovy" + + @Override + @Before + void setUp() throws Exception { + super.setUp() + } + + // TODO: needs fix for https://github.com/jenkinsci/JenkinsPipelineUnit/issues/419 +} diff --git a/tests/jenkins/TestHello.groovy b/tests/jenkins/TestHello.groovy new file mode 100644 index 000000000..de24220c3 --- /dev/null +++ b/tests/jenkins/TestHello.groovy @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import org.junit.* +import com.lesfurets.jenkins.unit.BasePipelineTest +import com.lesfurets.jenkins.unit.MethodCall +import static org.assertj.core.api.Assertions.assertThat + +class TestHello extends BasePipelineTest { + def jenkinsScript = "tests/jenkins/jobs/hello.groovy" + + @Override + @Before + void setUp() throws Exception { + super.setUp() + } + + @Test + void testHello() { + runScript(jenkinsScript) + assertJobStatusSuccess() + assertThat(helper.callStack.stream() + .filter { c -> c.methodName == "echo" } + .map(MethodCall.&callArgsToString) + .findAll { s -> s == "Hello World!" } + ).hasSize(1) + printCallStack() + } +} diff --git a/tests/jenkins/TestMessages.groovy b/tests/jenkins/TestMessages.groovy new file mode 100644 index 000000000..39f248ece --- /dev/null +++ b/tests/jenkins/TestMessages.groovy @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import org.junit.* +import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library +import static com.lesfurets.jenkins.unit.global.lib.ProjectSource.projectSource +import com.lesfurets.jenkins.unit.LibClassLoader +import com.lesfurets.jenkins.unit.declarative.* +import com.lesfurets.jenkins.unit.MethodCall +import static org.junit.Assert.* +import static org.assertj.core.api.Assertions.assertThat +import java.util.* + +class TestMessages extends DeclarativePipelineTest { + def jenkinsScript = "tests/jenkins/jobs/messages.groovy" + + @Override + @Before + void setUp() throws Exception { + super.setUp() + + helper.registerAllowedMethod('findFiles', [Map.class], null) + + helper.registerSharedLibrary( + library().name('jenkins') + .defaultVersion('') + .allowOverride(true) + .implicit(true) + .targetPath('') + .retriever(projectSource()) + .build() + ) + } + + @Test + void testMessages() throws Exception { + runScript(jenkinsScript) + + assertArrayEquals(Arrays.asList( + "{includes=messages/*, name=messages-stage1}", + "{includes=messages/*, name=messages-stage2}" + ).toArray(), helper.callStack.stream() + .filter { c -> c.methodName == "stash" } + .map(MethodCall.&callArgsToString) + .collect() + .toArray() + ) + + assertArrayEquals(Arrays.asList( + "{file=messages/stage1.msg, text=message 1}", + "{file=messages/stage2.msg, text=message 2}" + ).toArray(), helper.callStack.stream() + .filter { c -> c.methodName == "writeFile" } + .map(MethodCall.&callArgsToString) + .collect() + .toArray() + ) + + assertArrayEquals(Arrays.asList( + "{name=messages-stage1}", + "{name=messages-stage2}" + ).toArray(), helper.callStack.stream() + .filter { c -> c.methodName == "unstash" } + .map(MethodCall.&callArgsToString) + .collect() + .toArray() + ) + + assertThat(helper.callStack.stream() + .filter { c -> c.methodName == "deleteDir" } + .collect() + ).hasSize(1) + + assertJobStatusSuccess() + printCallStack() + } +} diff --git a/tests/jenkins/TestParallelMessages.groovy b/tests/jenkins/TestParallelMessages.groovy new file mode 100644 index 000000000..d13e559f0 --- /dev/null +++ b/tests/jenkins/TestParallelMessages.groovy @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import org.junit.* +import static org.junit.Assert.* +import java.util.* +import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library +import static com.lesfurets.jenkins.unit.global.lib.ProjectSource.projectSource +import com.lesfurets.jenkins.unit.LibClassLoader +import com.lesfurets.jenkins.unit.declarative.* +import com.lesfurets.jenkins.unit.MethodCall +import static org.assertj.core.api.Assertions.assertThat + +class TestParallelMessages extends DeclarativePipelineTest { + def jenkinsScript = "tests/jenkins/jobs/parallelmessages.groovy" + + @Override + @Before + void setUp() throws Exception { + super.setUp() + + helper.registerAllowedMethod('findFiles', [Map.class], null) + + helper.registerSharedLibrary( + library().name('jenkins') + .defaultVersion('') + .allowOverride(true) + .implicit(true) + .targetPath('') + .retriever(projectSource()) + .build() + ) + } + + @Test + @Ignore // raises MissingMethodException on docker agent declaration, need to upgrade jenkins-pipeline-unit which has new problems + void testParallelMessages() throws Exception { + binding.setVariable('scm', {}) + helper.registerAllowedMethod("legacySCM", [Closure.class], null) + + helper.registerAllowedMethod("library", [Map.class], { Map args -> + helper.getLibLoader().loadLibrary(args["identifier"]) + return new LibClassLoader(helper, null) + }) + + assertJobStatusSuccess() + printCallStack() + } +} diff --git a/tests/jenkins/jobs/Build.groovy b/tests/jenkins/jobs/Build.groovy new file mode 100644 index 000000000..af988e225 --- /dev/null +++ b/tests/jenkins/jobs/Build.groovy @@ -0,0 +1,180 @@ +// similar jenkins/opensearch/Jenkinsfile but echo instead of sh for testing + +lib = library(identifier: "jenkins@current", retriever: legacySCM(scm)).jenkins + +pipeline { + agent none + stages { + stage('parameters') { + steps { + script { + properties([ + parameters([ + string( + defaultValue: '2.0.0/opensearch-2.0.0.yml', + name: 'INPUT_MANIFEST', + trim: true + ) + ]) + ]) + } + } + } + stage('detect Docker image + args to use for the build') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028' + alwaysPull true + } + } + steps { + script { + git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' + manifest = readYaml(file: "manifests/$INPUT_MANIFEST") + + dockerImage = "${manifest.ci?.image?.name}" + // If the 'image' key is not present, it is populated with "null" string + if (dockerImage == null || dockerImage == "null") { + error("The Docker image for the build is required but was not provided in the manifest") + } + + dockerArgs = "${manifest.ci?.image?.args}" + // If the 'args' key is not present, it is populated with "null" string + if (dockerArgs == null || dockerArgs == "null") { + dockerArgs = '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot' + } + + echo "Using Docker image: " + dockerImage + echo "Using Docker container args: " + dockerArgs + } + } + } + stage('build') { + parallel { + stage('build-snapshots') { + environment { + SNAPSHOT_REPO_URL = "https://aws.oss.sonatype.org/content/repositories/snapshots/" + } + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image dockerImage + // Unlike freestyle docker, pipeline docker does not login to the container and run commands + // It use executes which does not source the docker container internal ENV VAR + args dockerArgs + alwaysPull true + } + } + steps { + script { + git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' + sh "echo ./build.sh manifests/$INPUT_MANIFEST --snapshot" + withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { + echo "$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/builds/maven" + } + } + } + post() { + always { + cleanWs disableDeferredWipeout: true, deleteDirs: true + } + } + } + stage('build-x64') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image dockerImage + // Unlike freestyle docker, pipeline docker does not login to the container and run commands + // It use executes which does not source the docker container internal ENV VAR + args dockerArgs + alwaysPull true + } + } + steps { + script { + build() + } + } + post() { + always { + cleanWs disableDeferredWipeout: true, deleteDirs: true + } + } + } + stage('build-arm64') { + agent { + docker { + label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + image dockerImage + // Unlike freestyle docker, pipeline docker does not login to the container and run commands + // It use executes which does not source the docker container internal ENV VAR + args dockerArgs + alwaysPull true + } + } + steps { + script { + build() + } + } + post() { + always { + cleanWs disableDeferredWipeout: true, deleteDirs: true + } + } + } + } + post() { + success { + node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + script { + def VERSION = sh(script: 'echo ${INPUT_MANIFEST} | grep -Po "[0-9.]+?(?=.yml)"', returnStdout: true).trim() + sh "echo VERSION:$VERSION BUILD_NUMBER:$BUILD_NUMBER" + def URL_x64 = "${PUBLIC_ARTIFACT_URL}/${env.JOB_NAME}/${VERSION}/${env.BUILD_NUMBER}/linux/x64/dist/opensearch-${VERSION}-linux-x64.tar.gz" + def URL_arm64 = "${PUBLIC_ARTIFACT_URL}/${env.JOB_NAME}/${VERSION}/${env.BUILD_NUMBER}/linux/arm64/dist/opensearch-${VERSION}-linux-arm64.tar.gz" + def s = "id && pwd && cd docker/release && curl -sSL $URL_x64 -o opensearch-x64.tgz && curl -sSL $URL_arm64 -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v ${VERSION} -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n ${env.BUILD_NUMBER}" + echo "dockerBuild:${s}" + def stashed = lib.Messages.new(this).get(['build-x64', 'build-arm64']) + publishNotification(":white_check_mark:", "Successful Build", "\n${stashed}") + } + } + } + failure { + node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + publishNotification(":warning:", "Failed Build", "") + } + } + } + } + } +} + +void build() { + git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' + + sh "echo ./build.sh manifests/$INPUT_MANIFEST" + + manifest = readYaml(file: 'tests/data/opensearch-build-1.1.0.yml') + + def artifactPath = "${env.JOB_NAME}/${manifest.build.version}/${env.BUILD_NUMBER}/${manifest.build.platform}/${manifest.build.architecture}"; + def BASE_URL = "${PUBLIC_ARTIFACT_URL}/${artifactPath}"; + + echo "sh ./assemble.sh builds/manifest.yml --base-url ${BASE_URL}" + + withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + echo "s3Upload(file: 'builds', bucket: '${ARTIFACT_BUCKET_NAME}', path: '${artifactPath}/builds')" + echo "s3Upload(file: 'dist', bucket: '${ARTIFACT_BUCKET_NAME}', path: '${artifactPath}/dist')" + } + + lib.Messages.new(this).add("${STAGE_NAME}", "${BASE_URL}/builds/manifest.yml\n${BASE_URL}/dist/manifest.yml") +} + +/** Publishes a notification to a slack instance*/ +void publishNotification(icon, msg, extra) { + withCredentials([string(credentialsId: 'BUILD_NOTICE_WEBHOOK', variable: 'TOKEN')]) { + def cmd = """curl -XPOST --header "Content-Type: application/json" --data '{"result_text": "$icon ${env.JOB_NAME} [${env.BUILD_NUMBER}] $msg ${env.BUILD_URL}\nManifest: ${INPUT_MANIFEST} $extra"}' """ + echo cmd + } +} \ No newline at end of file diff --git a/tests/jenkins/jobs/ParallelMessages.groovy b/tests/jenkins/jobs/ParallelMessages.groovy new file mode 100644 index 000000000..262e9591c --- /dev/null +++ b/tests/jenkins/jobs/ParallelMessages.groovy @@ -0,0 +1,79 @@ +lib = library(identifier: "jenkins@current", retriever: legacySCM(scm)) + +pipeline { + agent none + stages { + stage('build') { + parallel { + stage('build-on-host') { + steps { + node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + script { + def messages = lib.jenkins.Messages.new(this); + messages.add("${STAGE_NAME}", "built ${STAGE_NAME}") + } + } + } + } + stage('build-snapshots') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'ubuntu:latest' + alwaysPull + } + } + steps { + script { + build() + } + } + } + stage('build-x86') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'amazonlinux' + alwaysPull + } + } + steps { + script { + build() + } + } + } + stage('build-arm64') { + agent { + docker { + label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + image 'ubuntu:jammy' + alwaysPull + } + } + steps { + script { + build() + } + } + } + } + post() { + success { + node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + script { + def messages = lib.jenkins.Messages.new(this) + def stashed = messages.get(['build-on-host', 'build-snapshots', 'build-x86', 'build-arm64']) + echo stashed + } + } + } + } + } + } +} + +def build() { + def messages = lib.jenkins.Messages.new(this); + messages.add("${STAGE_NAME}", "built ${STAGE_NAME}") +} \ No newline at end of file diff --git a/tests/jenkins/jobs/hello.groovy b/tests/jenkins/jobs/hello.groovy new file mode 100644 index 000000000..2f5c1d21d --- /dev/null +++ b/tests/jenkins/jobs/hello.groovy @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +def greet = 'Hello World!' + +node() { + echo greet +} diff --git a/tests/jenkins/jobs/messages.groovy b/tests/jenkins/jobs/messages.groovy new file mode 100644 index 000000000..d6552ce9c --- /dev/null +++ b/tests/jenkins/jobs/messages.groovy @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +def lib = library("jenkins") + +pipeline { + agent none + stages { + stage('Example Build') { + steps { + def messages = lib.jenkins.Messages.new(this) + messages.add("stage1", "message 1") + messages.add("stage2", "message 2") + } + } + post() { + success { + script { + def messages = lib.jenkins.Messages.new(this) + def stashed = messages.get(["stage1", "stage2"]) + echo stashed + } + } + } + } +} \ No newline at end of file