Merge branch 'master' into discover-ssource-fields

Author: kibanamachine

.ci/es-snapshots/Jenkinsfile_build_es

@@ -25,90 +25,92 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
 timeout(time: 120, unit: 'MINUTES') {
   timestamps {
     ansiColor('xterm') {
-      node(workers.label('l')) {
-        catchErrors {
-          def VERSION
-          def SNAPSHOT_ID
-          def DESTINATION
-
-          def scmVars = checkoutEs(ES_BRANCH)
-          def GIT_COMMIT = scmVars.GIT_COMMIT
-          def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim()
-
-          buildArchives('to-archive')
-
-          dir('to-archive') {
-            def now = new Date()
-            def date = now.format("yyyyMMdd-HHmmss")
-
-            def filesRaw = sh(script: "ls -1", returnStdout: true).trim()
-            def files = filesRaw
-              .split("\n")
-              .collect { filename ->
-                // Filename examples
-                // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
-                // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
-                def parts = filename.replace("elasticsearch-oss", "oss").split("-")
-
-                VERSION = VERSION ?: parts[1]
-                SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}"
-                DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}"
-
-                return [
-                  filename: filename,
-                  checksum: filename + '.sha512',
-                  url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(),
-                  version: parts[1],
-                  platform: parts[3],
-                  architecture: parts[4].split('\\.')[0],
-                  license: parts[0] == 'oss' ? 'oss' : 'default',
+      slackNotifications.onFailure {
+        node(workers.label('l')) {
+          catchErrors {
+            def VERSION
+            def SNAPSHOT_ID
+            def DESTINATION
+
+            def scmVars = checkoutEs(ES_BRANCH)
+            def GIT_COMMIT = scmVars.GIT_COMMIT
+            def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim()
+
+            buildArchives('to-archive')
+
+            dir('to-archive') {
+              def now = new Date()
+              def date = now.format("yyyyMMdd-HHmmss")
+
+              def filesRaw = sh(script: "ls -1", returnStdout: true).trim()
+              def files = filesRaw
+                .split("\n")
+                .collect { filename ->
+                  // Filename examples
+                  // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
+                  // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
+                  def parts = filename.replace("elasticsearch-oss", "oss").split("-")
+
+                  VERSION = VERSION ?: parts[1]
+                  SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}"
+                  DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}"
+
+                  return [
+                    filename: filename,
+                    checksum: filename + '.sha512',
+                    url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(),
+                    version: parts[1],
+                    platform: parts[3],
+                    architecture: parts[4].split('\\.')[0],
+                    license: parts[0] == 'oss' ? 'oss' : 'default',
+                  ]
+                }
+
+              sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;'
+
+              def manifest = [
+                bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(),
+                branch: ES_BRANCH,
+                sha: GIT_COMMIT,
+                sha_short: GIT_COMMIT_SHORT,
+                version: VERSION,
+                generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")),
+                archives: files,
+              ]
+              def manifestJson = toJSON(manifest).toString()
+              writeFile file: 'manifest.json', text: manifestJson
+
+              upload(DESTINATION, '*.*')
+
+              sh "cp manifest.json manifest-latest.json"
+              upload(VERSION, 'manifest-latest.json')
+            }
+
+            if (PROMOTE_WITHOUT_VERIFY) {
+              esSnapshots.promote(VERSION, SNAPSHOT_ID)
+
+              emailext(
+                to: 'build-kibana@elastic.co',
+                subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}",
+                body: '${SCRIPT,template="groovy-html.template"}',
+                mimeType: 'text/html',
+              )
+            } else {
+              build(
+                propagate: false,
+                wait: false,
+                job: 'elasticsearch+snapshots+verify',
+                parameters: [
+                  string(name: 'branch_specifier', value: branch_specifier),
+                  string(name: 'SNAPSHOT_VERSION', value: VERSION),
+                  string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID),
                 ]
-              }
-
-            sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;'
-
-            def manifest = [
-              bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(),
-              branch: ES_BRANCH,
-              sha: GIT_COMMIT,
-              sha_short: GIT_COMMIT_SHORT,
-              version: VERSION,
-              generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")),
-              archives: files,
-            ]
-            def manifestJson = toJSON(manifest).toString()
-            writeFile file: 'manifest.json', text: manifestJson
-
-            upload(DESTINATION, '*.*')
-
-            sh "cp manifest.json manifest-latest.json"
-            upload(VERSION, 'manifest-latest.json')
+              )
+            }
           }
 
-          if (PROMOTE_WITHOUT_VERIFY) {
-            esSnapshots.promote(VERSION, SNAPSHOT_ID)
-
-            emailext(
-              to: 'build-kibana@elastic.co',
-              subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}",
-              body: '${SCRIPT,template="groovy-html.template"}',
-              mimeType: 'text/html',
-            )
-          } else {
-            build(
-              propagate: false,
-              wait: false,
-              job: 'elasticsearch+snapshots+verify',
-              parameters: [
-                string(name: 'branch_specifier', value: branch_specifier),
-                string(name: 'SNAPSHOT_VERSION', value: VERSION),
-                string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID),
-              ]
-            )
-          }
+          kibanaPipeline.sendMail()
         }
-
-        kibanaPipeline.sendMail()
       }
     }
   }

.ci/teamcity/setup_ci_stats.js

@@ -24,7 +24,7 @@ const ciStats = require('./ci_stats');
       branch: process.env.GIT_BRANCH.replace(/^(refs\/heads\/|origin\/)/, ''),
       commit: process.env.GIT_COMMIT,
       targetBranch: process.env.GITHUB_PR_TARGET_BRANCH || null,
-      mergeBase: null, // TODO
+      mergeBase: process.env.GITHUB_PR_MERGE_BASE || null,
     });
   } catch (ex) {
     console.error(ex);

.ci/teamcity/setup_env.sh

@@ -33,15 +33,17 @@ fi
 
 if is_pr; then
   tc_set_env ELASTIC_APM_ACTIVE false
-  tc_set_env CHECKS_REPORTER_ACTIVE true
+  tc_set_env CHECKS_REPORTER_ACTIVE "${CI_REPORTING_ENABLED-}"
 
   # These can be removed once we're not supporting Jenkins and TeamCity at the same time
   # These are primarily used by github checks reporter and can be configured via /github_checks_api.json
   tc_set_env ghprbGhRepository "elastic/kibana" # TODO?
   tc_set_env ghprbActualCommit "$GITHUB_PR_TRIGGERED_SHA"
   tc_set_env BUILD_URL "$TEAMCITY_BUILD_URL"
+
+  set_git_merge_base
 else
-  tc_set_env ELASTIC_APM_ACTIVE true
+  tc_set_env ELASTIC_APM_ACTIVE "${CI_REPORTING_ENABLED-}"
   tc_set_env CHECKS_REPORTER_ACTIVE false
 fi
 

.ci/teamcity/util.sh

@@ -79,3 +79,10 @@ tc_retry() {
   }
   tc_end_block "Retryable Step - Attempt #1"
 }
+
+set_git_merge_base() {
+  if [[ "${GITHUB_PR_TARGET_BRANCH-}" ]]; then
+    git fetch origin "$GITHUB_PR_TARGET_BRANCH"
+    tc_set_env GITHUB_PR_MERGE_BASE "$(git merge-base HEAD FETCH_HEAD)"
+  fi
+}

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,6 +1,7 @@
 ---
 name: Bug report
 about: Things break. Help us identify those things so we can fix them!
+labels: bug
 
 ---
 

.teamcity/src/Agents.kt

@@ -10,14 +10,16 @@ val StandardAgents = sizes.map { size -> size to GoogleCloudAgent {
   machineType = "n2-standard-$size"
   diskSizeGb = 75
   diskType = GoogleCloudAgentDiskType.SSD
+  maxInstances = 750
 } }.toMap()
 
 val BuildAgent = GoogleCloudAgent {
-    sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
-    agentPrefix = "kibana-c2-16-"
-    machineType = "c2-standard-16"
-    diskSizeGb = 250
-    diskType = GoogleCloudAgentDiskType.SSD
+  sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
+  agentPrefix = "kibana-c2-16-"
+  machineType = "c2-standard-16"
+  diskSizeGb = 250
+  diskType = GoogleCloudAgentDiskType.SSD
+  maxInstances = 200
 }
 
 val CloudProfile = GoogleCloudProfile {

.teamcity/src/Common.kt

@@ -0,0 +1,27 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.DslContext
+
+// If set to true, github check/commit status will be reported, failed-test-reporter will run, etc.
+const val ENABLE_REPORTING = false
+
+// If set to false, jobs with triggers (scheduled, on commit, etc) will be paused
+const val ENABLE_TRIGGERS = false
+
+fun getProjectBranch(): String {
+  return DslContext.projectName
+}
+
+fun getCorrespondingESBranch(): String {
+  return getProjectBranch().replace("_teamcity", "")
+}
+
+fun areTriggersEnabled(): Boolean {
+  return ENABLE_TRIGGERS;
+}
+
+fun isReportingEnabled(): Boolean {
+  return ENABLE_REPORTING;
+}
+
+fun makeSafeId(id: String): String {
+  return id.replace(Regex("[^a-zA-Z0-9_]"), "_")
+}

.teamcity/src/Extensions.kt

@@ -39,7 +39,9 @@ val testArtifactRules = """
 fun BuildType.addTestSettings() {
   artifactRules += "\n" + testArtifactRules
   steps {
-    failedTestReporter()
+    if(isReportingEnabled()) {
+       failedTestReporter()
+    }
   }
   features {
     junit()

.teamcity/src/builds/BaselineCi.kt

@@ -1,10 +1,12 @@
 package builds
 
 import addSlackNotifications
+import areTriggersEnabled
 import builds.default.DefaultBuild
 import builds.default.DefaultSavedObjectFieldMetrics
 import builds.oss.OssBuild
 import dependsOn
+import getProjectBranch
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
 import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
 import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs
@@ -15,14 +17,14 @@ object BaselineCi : BuildType({
   name = "Baseline CI"
   description = "Runs builds, saved object field metrics for every commit"
   type = Type.COMPOSITE
-  paused = true
+  paused = !areTriggersEnabled()
 
   templates(KibanaTemplate)
 
   triggers {
     vcs {
-      branchFilter = "refs/heads/master_teamcity"
-//      perCheckinTriggering = true // TODO re-enable this later, it wreaks havoc when I merge upstream
+      branchFilter = "refs/heads/${getProjectBranch()}"
+      perCheckinTriggering = areTriggersEnabled()
     }
   }
 

.teamcity/src/builds/HourlyCi.kt

@@ -1,7 +1,9 @@
 package builds
 
 import addSlackNotifications
+import areTriggersEnabled
 import dependsOn
+import getProjectBranch
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
 import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
 import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.schedule
@@ -11,14 +13,15 @@ object HourlyCi : BuildType({
   name = "Hourly CI"
   description = "Runs everything in CI, hourly"
   type = Type.COMPOSITE
+  paused = !areTriggersEnabled()
 
   triggers {
     schedule {
       schedulingPolicy = cron {
         hours = "*"
         minutes = "0"
       }
-      branchFilter = "refs/heads/master_teamcity"
+      branchFilter = "refs/heads/${getProjectBranch()}"
       triggerBuild = always()
       withPendingChangesOnly = true
     }