Merge remote-tracking branch 'upstream/7.9' into backport/7.9/pr-19960

* upstream/7.9: (32 commits) feat(ci): support storing artifacts for PRs in separate dirs (elastic#20282) (elastic#20301) Cisco ASA: Fix message 106100 (elastic#20245) (elastic#20277) [CI] Change upstream reference (elastic#20296) (elastic#20297) [docs] Fix Windows download link for agent (elastic#20258) (elastic#20290) Cherry-pick to 7.9: [docs] Rename release highlights to what's new (elastic#20255) (elastic#20285) Elastic agent on k8s (elastic#19727) (elastic#20262) [Filebeat Module] Defender ATP - Adding dashboard (elastic#20058) (elastic#20093) fix: use a fixed worker type for tests (elastic#20130) (elastic#20247) [Elastic Agent] Fix Windows powershell install service script (elastic#20203) (elastic#20252) [Ingest Manager] Fixed unzip on older windows (elastic#20088) (elastic#20109) adding possibility to override content-type checks, it was breaking certain webhooks that is not able to set content-headers at all. Still defaults to application/json (elastic#20232) (elastic#20237) [Filebeat][Gsuite] Make GSuite docs more clear (elastic#19981) (elastic#20067) Increase index.max_docvalue_fields_search to 200 (elastic#20218) (elastic#20221) Call host parser only once when building light metricsets (elastic#20149) (elastic#20190) [Metricbeat] Use MySQL Host Parser in Query metricset (elastic#20191) (elastic#20212) [Filebeat] Ignore cylance.protect timestamps while testing (elastic#20207) (elastic#20217) [libbeat] Fix write error in ensureWriter.Write (elastic#20112) (elastic#20145) Cherry-pick elastic#20127 to 7.9: Fix failing unit tests on windows (elastic#20180) Remove f5/firepass rsa2elk fileset (elastic#20160) (elastic#20206) Cherry-pick elastic#20138 to 7.9: [Filebeat] Update crowdstrike module (elastic#20177) ...
v1v · Jul 29, 2020 · 59a6124 · 59a6124
2 parents 244f01f + 8a48db7
commit 59a6124
Show file tree

Hide file tree

Showing 212 changed files with 9,584 additions and 15,168 deletions.
diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy
@@ -27,7 +27,7 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
-    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+    upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   stages {
     stage('Filter build') {

diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -26,7 +26,7 @@ pipeline {
   triggers {
     issueCommentTrigger('(?i)^\\/packag[ing|e]$')
     // disable upstream trigger on a PR basis
-    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+    upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
@@ -222,7 +222,12 @@ def withMacOSEnv(Closure body){
 }
 
 def publishPackages(baseDir){
-  googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/snapshots",
+  def bucketUri = "gs://${JOB_GCS_BUCKET}/snapshots"
+  if (env.CHANGE_ID?.trim()) {
+    bucketUri = "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}"
+  }
+
+  googleStorageUpload(bucket: "${bucketUri}",
     credentialsId: "${JOB_GCS_CREDENTIALS}",
     pathPrefix: "${baseDir}/build/distributions/",
     pattern: "${baseDir}/build/distributions/**/*",

diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -52,6 +52,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 ==== Bugfixes
 
 - Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}14162[14162]
+- Metricbeat module builders call host parser only once when instantiating light modules. {pull}20149[20149]
 
 ==== Added
 

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -178,7 +178,6 @@ field. You can revert this change by configuring tags for the module and omittin
 - Add experimental dataset cisco/nexus for Cisco Nexus logs {pull}19713[19713]
 - Add experimental dataset citrix/virtualapps for Citrix Virtual Apps logs {pull}19713[19713]
 - Add experimental dataset cylance/protect for Cylance Protect logs {pull}19713[19713]
-- Add experimental dataset f5/firepass for F5 FirePass SSL VPN logs {pull}19713[19713]
 - Add experimental dataset fortinet/clientendpoint for Fortinet FortiClient Endpoint Protection logs {pull}19713[19713]
 - Add experimental dataset imperva/securesphere for Imperva Secure Sphere logs {pull}19713[19713]
 - Add experimental dataset infoblox/nios for Infoblox Network Identity Operating System logs {pull}19713[19713]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -89,7 +89,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix goroutine leak and Elasticsearch output file descriptor leak when output reloading is in use. {issue}10491[10491] {pull}17381[17381]
 - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030]
 - Change `decode_json_fields` processor, to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
+- Fix terminating pod autodiscover issue. {pull}20084[20084]
 - Fix seccomp policy for calls to `chmod` and `chown`. {pull}20054[20054]
+- Output errors when Kibana index pattern setup fails. {pull}20121[20121]
 
 *Auditbeat*
 
@@ -121,6 +123,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed typo in log message. {pull}17897[17897]
 - Fix S3 input to trim delimiter /n from each log line. {pull}19972[19972]
 - Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962]
+- Ignore missing in Zeek module when dropping unnecessary fields. {pull}19984[19984]
+- Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138]
+- Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245]
 
 *Heartbeat*
 
@@ -193,6 +198,19 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
+- Add backoff configuration options for the Kafka output. {issue}16777[16777] {pull}17808[17808]
+- Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607]
+- Change ownership of files in docker images so they can be used in secured environments. {pull}12905[12905]
+- Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817]
+- Add support for multiple sets of hints on autodiscover {pull}18883[18883]
+- Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181]
+- Add data type conversion in `dissect` processor for converting string values to other basic data types. {pull}18683[18683]
+- Add the `ignore_failure` configuration option to the dissect processor. {pull}19464[19464]
+- Add the `overwrite_keys` configuration option to the dissect processor. {pull}19464[19464]
+- Add support to trim captured values in the dissect processor. {pull}19464[19464]
+- Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
+- Add support for DNS over TLS for the dns_processor. {pull}19321[19321]
+- Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
 
 *Auditbeat*
 
@@ -235,6 +253,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added an input option `publisher_pipeline.disable_host` to disable `host.name`
 from being added to events by default. {pull}18159[18159]
 - Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
+- Add event.ingested for CrowdStrike module {pull}20138[20138]
+- Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138]
 
 *Heartbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -10,7 +10,7 @@ import groovy.transform.Field
 @Field def stashedTestReports = [:]
 
 pipeline {
-  agent { label 'ubuntu && immutable' }
+  agent { label 'ubuntu-18 && immutable' }
   environment {
     BASE_DIR = 'src/github.com/elastic/beats'
     GOX_FLAGS = "-arch amd64"
@@ -88,7 +88,7 @@ pipeline {
       failFast false
       parallel {
         stage('Elastic Agent x-pack'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -135,7 +135,7 @@ pipeline {
         }
 
         stage('Filebeat oss'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -148,7 +148,7 @@ pipeline {
           }
         }
         stage('Filebeat x-pack'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -223,7 +223,7 @@ pipeline {
           }
         }
         stage('Heartbeat'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -271,7 +271,7 @@ pipeline {
           }
         }
         stage('Auditbeat oss Linux'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -284,7 +284,7 @@ pipeline {
           }
         }
         stage('Auditbeat crosscompile'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -328,7 +328,7 @@ pipeline {
           }
         }
         stage('Auditbeat x-pack'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -367,7 +367,7 @@ pipeline {
           }
         }
         stage('Libbeat'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -394,7 +394,7 @@ pipeline {
           }
         }
         stage('Libbeat x-pack'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -407,7 +407,7 @@ pipeline {
           }
         }
         stage('Metricbeat OSS Unit tests'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -419,8 +419,8 @@ pipeline {
             mageTarget(context: "Metricbeat OSS linux/amd64 (unitTest)", directory: "metricbeat", target: "build unitTest")
           }
         }
-        stage('Metricbeat OSS Go Integration tests'){
-          agent { label 'ubuntu && immutable' }
+        stage('Metricbeat OSS Integration tests'){
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -432,8 +432,8 @@ pipeline {
             mageTarget(context: "Metricbeat OSS linux/amd64 (goIntegTest)", directory: "metricbeat", target: "goIntegTest", withModule: true)
           }
         }
-        stage('Metricbeat OSS Python Integration tests'){
-          agent { label 'ubuntu && immutable' }
+        stage('Metricbeat Python integration tests'){
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -446,7 +446,7 @@ pipeline {
           }
         }
         stage('Metricbeat x-pack'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -456,7 +456,7 @@ pipeline {
           }
           stages {
             stage('Prepare cloud integration tests environments'){
-              agent { label 'ubuntu && immutable' }
+              agent { label 'ubuntu-18 && immutable' }
               options { skipDefaultCheckout() }
               steps {
                 startCloudTestEnv('x-pack-metricbeat', [
@@ -465,7 +465,7 @@ pipeline {
               }
             }
             stage('Metricbeat x-pack'){
-              agent { label 'ubuntu && immutable' }
+              agent { label 'ubuntu-18 && immutable' }
               options { skipDefaultCheckout() }
               steps {
                 withCloudTestEnv() {
@@ -481,7 +481,7 @@ pipeline {
           }
         }
         stage('Metricbeat crosscompile'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -550,8 +550,8 @@ pipeline {
             mageTargetWin(context: "Metricbeat x-pack Windows", directory: "x-pack/metricbeat", target: "build unitTest")
           }
         }
-        stage('Packetbeat OSS'){
-          agent { label 'ubuntu && immutable' }
+        stage('Packetbeat'){
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -599,7 +599,7 @@ pipeline {
           }
         }
         stage('dockerlogbeat'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -616,7 +616,7 @@ pipeline {
           }
         }
         stage('Winlogbeat'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -659,7 +659,7 @@ pipeline {
           }
         }
         stage('Functionbeat'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -710,7 +710,7 @@ pipeline {
           }
         }
         stage('Journalbeat'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -727,7 +727,7 @@ pipeline {
           }
         }
         stage('Generators'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -787,7 +787,7 @@ pipeline {
           }
         }
         stage('Kubernetes'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           when {
             beforeAgent true