Merge branch '7.x' into backport_17881_7.x

.ci/packaging.groovy

@@ -125,11 +125,51 @@ pipeline {
 }
 
 def pushCIDockerImages(){
-  sh(label: 'Push Docker image', script: '''
-    if [ -n "$(command -v docker)" ]; then
-      docker images || true
-    fi
-  ''')
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    if ("${env.BEATS_FOLDER}" == "auditbeat"){
+      tagAndPush('auditbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "filebeat") {
+      tagAndPush('filebeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "heartbeat"){
+      tagAndPush('heartbeat')
+      tagAndPush('heartbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
+      tagAndPush('journalbeat')
+      tagAndPush('journalbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "metricbeat"){
+      tagAndPush('metricbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "packetbeat"){
+      tagAndPush('packetbeat')
+      tagAndPush('packetbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/auditbeat"){
+      tagAndPush('auditbeat')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
+      tagAndPush('elastic-agent')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
+      tagAndPush('filebeat')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
+      tagAndPush('metricbeat')
+    }
+  }
+}
+
+def tagAndPush(name){
+  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+  if("${env.SNAPSHOT}" == "true"){
+    libbetaVer += "-SNAPSHOT"
+  }
+  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}"
+  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
+  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+  retry(3){
+    sh(label:'Change tag and push', script: """
+      docker tag ${oldName} ${newName}
+      docker push ${newName}
+      docker tag ${oldName} ${commitName}
+      docker push ${commitName}
+    """)
+  }
 }
 
 def release(){

CHANGELOG-developer.next.asciidoc

@@ -36,6 +36,11 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Extract Elasticsearch client logic from `outputs/elasticsearch` package into new `esclientleg` package. {pull}16150[16150]
 - Rename `queue.BufferConfig.Events` to `queue.BufferConfig.MaxEvents`. {pull}17622[17622]
 - Remove `queue.Feature` and replace `queue.RegisterType` with `queue.RegisterQueueType`. {pull}17666[17666]
+- The way configuration files are generated has changed to make it easier to customize parts
+  of the config without requiring changes to libbeat config templates. Generation is now
+  fully based on Go text/template and no longer uses file concatenation to generate the config.
+  Your magefile.go will require a change to adapt the devtool API. See the pull request for
+  more details. {pull}18148[18148]
 - Introduce APM libbeat instrumentation. `Publish` method on `Client` interface now takes a Context as first argument. {pull}17938[17938]
 
 ==== Bugfixes

CHANGELOG.next.asciidoc

@@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix goroutine leak and Elasticsearch output file descriptor leak when output reloading is in use. {issue}10491[10491] {pull}17381[17381]
 - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030]
 - Fix panic when assigning a key to a `nil` value in an event. {pull}18143[18143]
+- Change `decode_json_fields` processor, to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
 
 *Auditbeat*
 
@@ -274,6 +275,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153]
+- When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 
 *Auditbeat*
 
@@ -283,6 +285,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix syscall kprobe arguments for 32-bit systems in socket module. {pull}17500[17500]
 - Fix memory leak on when we miss socket close kprobe events. {pull}17500[17500]
 - Add system module process dataset ECS categorization fields. {pull}18032[18032]
+- Add system module user dataset ECS categorization fields. {pull}18035[18035]
+- Add system module login dataset ECS categorization fields. {pull}18034[18034]
+- Add system module package dataset ECS categorization fields. {pull}18033[18033]
+- Add ECS categories for system module host dataset. {pull}18031[18031]
+- Add system module socket dataset ECS categorization fields. {pull}18036[18036]
 
 *Filebeat*
 
@@ -356,14 +363,18 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added Unix stream socket support as an input source and a syslog input source. {pull}17492[17492]
 - Improve ECS categorization field mappings in misp module. {issue}16026[16026] {pull}17344[17344]
 - Enhance `elasticsearch/deprecation` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17728[17728]
+- Make `decode_cef` processor GA. {pull}17944[17944]
 - Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918]
 - Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916]
 - Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
 - Improve ECS categorization field mappings for nginx module. {issue}16174[16174] {pull}17844[17844]
 - Add support for Google Application Default Credentials to the Google Pub/Sub input and Google Cloud modules. {pull}15668[15668]
 - Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
 - Improve ECS categorization field mappings for netflow module. {issue}16135[16135] {pull}18108[18108]
+- Added an input option `publisher_pipeline.disable_host` to disable `host.name`
+  from being added to events by default. {pull}18159[18159]
 - Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065]
+- Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
 - Improve ECS categorization field mappings in osquery module. {issue}16176[16176] {pull}17881[17881]
 
 *Heartbeat*
@@ -468,6 +479,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add experimental event log reader implementation that should be faster in most cases. {issue}6585[6585] {pull}16849[16849]
 - Set process.command_line and process.parent.command_line from Sysmon Event ID 1. {pull}17327[17327]
 - Add support for event IDs 4673,4674,4697,4698,4699,4700,4701,4702,4768,4769,4770,4771,4776,4778,4779,4964 to the Security module {pull}17517[17517]
+- Add registry and code signature information and ECS categorization fields for sysmon module {pull}18058[18058]
 
 ==== Deprecated
 

Jenkinsfile

@@ -906,7 +906,7 @@ def isChangedOSSCode(patterns) {
     "^libbeat/.*",
     "^testing/.*",
     "^dev-tools/.*",
-    "^\\.ci/.*",
+    "^\\.ci/scripts/.*",
   ]
   allPatterns.addAll(patterns)
   return isChanged(allPatterns)
@@ -920,7 +920,7 @@ def isChangedXPackCode(patterns) {
     "^dev-tools/.*",
     "^testing/.*",
     "^x-pack/libbeat/.*",
-    "^\\.ci/.*",
+    "^\\.ci/scripts/.*",
   ]
   allPatterns.addAll(patterns)
   return isChanged(allPatterns)

NOTICE.txt

@@ -6580,8 +6580,7 @@ SoundCloud Ltd. (http://soundcloud.com/).
 
 --------------------------------------------------------------------
 Dependency: github.com/prometheus/procfs
-Version: v0.0.9
-Revision: 42f6e295b56f
+Version: v0.0.11
 License type (autodetected): Apache-2.0
 ./vendor/github.com/prometheus/procfs/LICENSE:
 --------------------------------------------------------------------
@@ -7823,7 +7822,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/sys
-Revision: c96a22e43c9c
+Revision: b016eb3dc98e
 License type (autodetected): BSD-3-Clause
 ./vendor/golang.org/x/sys/LICENSE:
 --------------------------------------------------------------------

auditbeat/_meta/config/auditbeat.config.modules.yml.tmpl

@@ -0,0 +1,18 @@
+{{header "Config Reloading"}}
+
+# Config reloading allows to dynamically load modules. Each file which is
+# monitored must contain one or multiple modules as a list.
+auditbeat.config.modules:
+
+  # Glob pattern for configuration reloading
+  path: ${path.config}/modules.d/*.yml
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+# Maximum amount of time to randomly delay the start of a dataset. Use 0 to
+# disable startup delay.
+auditbeat.max_start_delay: 10s

auditbeat/_meta/config/auditbeat.modules.yml.tmpl

@@ -0,0 +1,2 @@
+{{header "Modules configuration"}}
+auditbeat.modules:

auditbeat/_meta/config/beat.reference.yml.tmpl

@@ -0,0 +1,4 @@
+{{template "header.reference.yml.tmpl" .}}
+{{template "auditbeat.config.modules.yml.tmpl" .}}
+{{template "auditbeat.modules.yml.tmpl" .}}
+{{template "config.modules.yml.tmpl" .}}

auditbeat/_meta/config/beat.yml.tmpl

@@ -0,0 +1,4 @@
+{{template "header.yml.tmpl" .}}
+{{template "auditbeat.modules.yml.tmpl" .}}
+{{template "config.modules.yml.tmpl" .}}
+{{template "setup.template.yml.tmpl" .}}

auditbeat/_meta/config/header.reference.yml.tmpl

@@ -0,0 +1,8 @@
+########################## Auditbeat Configuration #############################
+
+# This is a reference configuration file documenting all non-deprecated options
+# in comments. For a shorter configuration example that contains only the most
+# common options, please see auditbeat.yml in the same directory.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/auditbeat/index.html

auditbeat/_meta/common.p1.yml → auditbeat/_meta/config/header.yml.tmpl

@@ -6,7 +6,3 @@
 #
 # You can find the full configuration reference here:
 # https://www.elastic.co/guide/en/beats/auditbeat/index.html
-
-#==========================  Modules configuration =============================
-auditbeat.modules:
-

auditbeat/_meta/config/setup.template.yml.tmpl

@@ -0,0 +1,5 @@
+{{header "Elasticsearch template setting"}}
+setup.template.settings:
+  index.number_of_shards: 1
+  #index.codec: best_compression
+  #_source.enabled: false

auditbeat/auditbeat.docker.yml

@@ -12,6 +12,7 @@ auditbeat.modules:
     - /sbin
     - /usr/sbin
     - /etc
+
 processors:
 - add_cloud_metadata: ~
 - add_docker_metadata: ~