|
| 1 | +#!/bin/groovy |
| 2 | + |
| 3 | +// This job effectively has two SCM configurations: |
| 4 | +// one for kibana, used to check out this Jenkinsfile (which means it's the job's main SCM configuration), as well as kick-off the downstream verification job |
| 5 | +// one for elasticsearch, used to check out the elasticsearch source before building it |
| 6 | + |
| 7 | +// There are two parameters that drive which branch is checked out for each of these, but they will typically be the same |
| 8 | +// 'branch_specifier' is for kibana / the job itself |
| 9 | +// ES_BRANCH is for elasticsearch |
| 10 | + |
| 11 | +library 'kibana-pipeline-library' |
| 12 | +kibanaLibrary.load() |
| 13 | + |
| 14 | +def ES_BRANCH = params.ES_BRANCH |
| 15 | + |
| 16 | +if (!ES_BRANCH) { |
| 17 | + error "Parameter 'ES_BRANCH' must be specified." |
| 18 | +} |
| 19 | + |
| 20 | +currentBuild.displayName += " - ${ES_BRANCH}" |
| 21 | +currentBuild.description = "ES: ${ES_BRANCH}<br />Kibana: ${params.branch_specifier}" |
| 22 | + |
| 23 | +def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION |
| 24 | + |
| 25 | +timeout(time: 120, unit: 'MINUTES') { |
| 26 | + timestamps { |
| 27 | + ansiColor('xterm') { |
| 28 | + node('linux && immutable') { |
| 29 | + catchError { |
| 30 | + def VERSION |
| 31 | + def SNAPSHOT_ID |
| 32 | + def DESTINATION |
| 33 | + |
| 34 | + def scmVars = checkoutEs(ES_BRANCH) |
| 35 | + def GIT_COMMIT = scmVars.GIT_COMMIT |
| 36 | + def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() |
| 37 | + |
| 38 | + buildArchives('to-archive') |
| 39 | + |
| 40 | + dir('to-archive') { |
| 41 | + def now = new Date() |
| 42 | + def date = now.format("yyyyMMdd-HHmmss") |
| 43 | + |
| 44 | + def filesRaw = sh(script: "ls -1", returnStdout: true).trim() |
| 45 | + def files = filesRaw |
| 46 | + .split("\n") |
| 47 | + .collect { filename -> |
| 48 | + // Filename examples |
| 49 | + // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz |
| 50 | + // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz |
| 51 | + def parts = filename.replace("elasticsearch-oss", "oss").split("-") |
| 52 | + |
| 53 | + VERSION = VERSION ?: parts[1] |
| 54 | + SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}" |
| 55 | + DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}" |
| 56 | + |
| 57 | + return [ |
| 58 | + filename: filename, |
| 59 | + checksum: filename + '.sha512', |
| 60 | + url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(), |
| 61 | + version: parts[1], |
| 62 | + platform: parts[3], |
| 63 | + architecture: parts[4].split('\\.')[0], |
| 64 | + license: parts[0] == 'oss' ? 'oss' : 'default', |
| 65 | + ] |
| 66 | + } |
| 67 | + |
| 68 | + sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' |
| 69 | + |
| 70 | + def manifest = [ |
| 71 | + bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(), |
| 72 | + branch: ES_BRANCH, |
| 73 | + sha: GIT_COMMIT, |
| 74 | + sha_short: GIT_COMMIT_SHORT, |
| 75 | + version: VERSION, |
| 76 | + generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")), |
| 77 | + archives: files, |
| 78 | + ] |
| 79 | + def manifestJson = toJSON(manifest).toString() |
| 80 | + writeFile file: 'manifest.json', text: manifestJson |
| 81 | + |
| 82 | + upload(DESTINATION, '*.*') |
| 83 | + |
| 84 | + sh "cp manifest.json manifest-latest.json" |
| 85 | + upload(VERSION, 'manifest-latest.json') |
| 86 | + } |
| 87 | + |
| 88 | + if (PROMOTE_WITHOUT_VERIFY) { |
| 89 | + esSnapshots.promote(VERSION, SNAPSHOT_ID) |
| 90 | + |
| 91 | + emailext( |
| 92 | + to: 'build-kibana@elastic.co', |
| 93 | + subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}", |
| 94 | + body: '${SCRIPT,template="groovy-html.template"}', |
| 95 | + mimeType: 'text/html', |
| 96 | + ) |
| 97 | + } else { |
| 98 | + build( |
| 99 | + propagate: false, |
| 100 | + wait: false, |
| 101 | + job: 'elasticsearch+snapshots+verify', |
| 102 | + parameters: [ |
| 103 | + string(name: 'branch_specifier', value: branch_specifier), |
| 104 | + string(name: 'SNAPSHOT_VERSION', value: VERSION), |
| 105 | + string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID), |
| 106 | + ] |
| 107 | + ) |
| 108 | + } |
| 109 | + } |
| 110 | + |
| 111 | + kibanaPipeline.sendMail() |
| 112 | + } |
| 113 | + } |
| 114 | + } |
| 115 | +} |
| 116 | + |
| 117 | +def checkoutEs(branch) { |
| 118 | + retryWithDelay(8, 15) { |
| 119 | + return checkout([ |
| 120 | + $class: 'GitSCM', |
| 121 | + branches: [[name: branch]], |
| 122 | + doGenerateSubmoduleConfigurations: false, |
| 123 | + extensions: [], |
| 124 | + submoduleCfg: [], |
| 125 | + userRemoteConfigs: [[ |
| 126 | + credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', |
| 127 | + url: 'git@github.com:elastic/elasticsearch', |
| 128 | + ]], |
| 129 | + ]) |
| 130 | + } |
| 131 | +} |
| 132 | + |
| 133 | +def upload(destination, pattern) { |
| 134 | + return googleStorageUpload( |
| 135 | + credentialsId: 'kibana-ci-gcs-plugin', |
| 136 | + bucket: "gs://kibana-ci-es-snapshots-daily/${destination}", |
| 137 | + pattern: pattern, |
| 138 | + sharedPublicly: false, |
| 139 | + showInline: false, |
| 140 | + ) |
| 141 | +} |
| 142 | + |
| 143 | +def buildArchives(destination) { |
| 144 | + def props = readProperties file: '.ci/java-versions.properties' |
| 145 | + withEnv([ |
| 146 | + // Select the correct JDK for this branch |
| 147 | + "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", |
| 148 | + |
| 149 | + // These Jenkins env vars trigger some automation in the elasticsearch repo that we don't want |
| 150 | + "BUILD_NUMBER=", |
| 151 | + "JENKINS_URL=", |
| 152 | + "BUILD_URL=", |
| 153 | + "JOB_NAME=", |
| 154 | + "NODE_NAME=", |
| 155 | + ]) { |
| 156 | + sh """ |
| 157 | + ./gradlew -p distribution/archives assemble --parallel |
| 158 | + mkdir -p ${destination} |
| 159 | + find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\; |
| 160 | + """ |
| 161 | + } |
| 162 | +} |
0 commit comments