Merge branch 'master' into feature/xp/mb/update-mssql

.ci/packaging.groovy

@@ -23,6 +23,7 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i)^\\/packaging$')
+    upstream('Beats/beats-beats-mbp/master')
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
@@ -76,7 +77,19 @@ pipeline {
             }
             environment {
               HOME = "${env.WORKSPACE}"
-              PLATFORMS = "!darwin +linux/armv7 +linux/ppc64le +linux/s390x +linux/mips64"
+              PLATFORMS = [
+                '+all',
+                'linux/amd64',
+                'linux/386',
+                'linux/arm64',
+                'linux/armv7',
+                'linux/ppc64le',
+                'linux/mips64',
+                'linux/s390x',
+                'windows/amd64',
+                'windows/386',
+                (params.macos ? '' : 'darwin/amd64'),
+              ].join(' ')
             }
             steps {
               release()
@@ -94,7 +107,10 @@ pipeline {
             }
             environment {
               HOME = "${env.WORKSPACE}"
-              PLATFORMS = "!defaults +darwin/amd64"
+              PLATFORMS = [
+                '+all',
+                'darwin/amd64',
+              ].join(' ')
             }
             steps {
               withMacOSEnv(){

.ci/scripts/install-kind.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+DEFAULT_HOME="/usr/local"
+KIND_VERSION=${KIND_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KIND_CMD="${HOME}/bin/kind"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+chmod +x "${KIND_CMD}"

.ci/scripts/install-kubectl.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+DEFAULT_HOME="/usr/local"
+K8S_VERSION=${K8S_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KUBECTL_CMD="${HOME}/bin/kubectl"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KUBECTL_CMD}"
+

.ci/scripts/kind-setup.sh

@@ -1,18 +1,5 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-MSG="parameter missing."
-K8S_VERSION=${K8S_VERSION:?$MSG}
-HOME=${HOME:?$MSG}
-KBC_CMD="${HOME}/bin/kubectl"
-
-mkdir -p "${HOME}/bin"
-
-curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-chmod +x "${KBC_CMD}"
-
-GO111MODULE="on" go get sigs.k8s.io/kind@v0.5.1
 kind create cluster --image kindest/node:${K8S_VERSION}
-
-export KUBECONFIG="$(kind get kubeconfig-path)"
 kubectl cluster-info

.travis.yml

@@ -111,23 +111,28 @@ jobs:
     # Metricbeat
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="-C metricbeat unit-tests coverage-report"
+      env: TARGETS="-C metricbeat unit-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat integration-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      install: 
+        - .ci/scripts/install-kind.sh
+        - .ci/scripts/install-kubectl.sh
+      env:
+        - TARGETS="-C metricbeat integration-tests"
+        - K8S_VERSION=v1.17.2
+        - KIND_VERSION=v0.7.0
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat update system-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat system-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
-
     - os: osx
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="TEST_ENVIRONMENT=0 -C metricbeat testsuite"
+      env: TARGETS="-C metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
@@ -137,6 +142,21 @@ jobs:
       stage: test
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat unit-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat integration-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat system-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
       env: TARGETS="-C x-pack/metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test

CHANGELOG.asciidoc

@@ -30,6 +30,7 @@ https://github.com/elastic/beats/compare/v7.5.0...v7.5.1[View commits]
 - Fix docker network stats when multiple interfaces are configured. {issue}14586[14586] {pull}14825[14825]
 - Fix ListMetrics pagination in aws module. {issue}14926[14926] {pull}14942[14942]
 - Fix CPU count in docker/cpu in cases where no `online_cpus` are reported {pull}15070[15070]
+- Add domain state to kvm module {pull}17673[17673]
 
 [[release-notes-7.5.0]]
 === Beats version 7.5.0

CHANGELOG.next.asciidoc

@@ -20,7 +20,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630]
 
 *Filebeat*
-
+- Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
+- Improve ECS categorization field mappings for nginx module. http.request.referrer is now lowercase & http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
+- Improve ECS field mappings in santa module. move hash.sha256 to process.hash.sha256 & move certificate fields to santa.certificate . {issue}16180[16180] {pull}17982[17982]
 
 *Heartbeat*
 
@@ -75,6 +77,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix bug with `monitoring.cluster_uuid` setting not always being exposed via GET /state Beats API. {issue}16732[16732] {pull}17420[17420]
 - Fix building on FreeBSD by removing build flags from `add_cloudfoundry_metadata` processor. {pull}17486[17486]
 - Do not rotate log files on startup when interval is configured and rotateonstartup is disabled. {pull}17613[17613]
+- Fix goroutine leak and Elasticsearch output file descriptor leak when output reloading is in use. {issue}10491[10491] {pull}17381[17381]
+- Fix `setup.dashboards.index` setting not working. {pull}17749[17749]
+- Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030]
 
 *Auditbeat*
 
@@ -111,6 +116,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed activemq module causing "regular expression has redundant nested repeat operator" warning in Elasticsearch. {pull}17428[17428]
 - Remove migrationVersion map 7.7.0 reference from Kibana dashboard file to fix backward compatibility issues. {pull}17425[17425]
 - Fix issue 17734 to retry on rate-limit error in the Filebeat httpjson input. {issue}17734[17734] {pull}17735[17735]
+- Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847]
+- Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834]
+- Fixed typo in log message. {pull}17897[17897]
+- Fix Cisco ASA ASA 3020** and 106023 messages {pull}17964[17964]
 
 *Heartbeat*
 
@@ -194,9 +203,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update RPM packages contained in Beat Docker images. {issue}17035[17035]
 - Update supported versions of `redis` output. {pull}17198[17198]
 - Update documentation for system.process.memory fields to include clarification on Windows os's. {pull}17268[17268]
+- Add `replace` processor for replacing string values of fields. {pull}17342[17342]
 - Add optional regex based cid extractor to `add_kubernetes_metadata` processor. {pull}17360[17360]
 - Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505]
 - Add support for AWS IAM `role_arn` in credentials config. {pull}17658[17658] {issue}12464[12464]
+- Add keystore support for autodiscover static configurations. {pull]16306[16306]
+- Add Kerberos support to Elasticsearch output. {pull}17927[17927]
 
 *Auditbeat*
 
@@ -257,6 +269,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379]
 - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491]
 - Release Google Cloud module as GA. {pull}17511[17511]
+- Add config option to select a different azure cloud env in the azure-eventhub input and azure module. {issue}17649[17649] {pull}17659[17659]
 - Added new Checkpoint Syslog filebeat module. {pull}17682[17682]
 - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550]
 - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714]
@@ -265,11 +278,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729]
 - Improve ECS categorization field mappings in misp module. {issue}16026[16026] {pull}17344[17344]
 - Added Unix stream socket support as an input source and a syslog input source. {pull}17492[17492]
+- Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
+- Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916]
+- Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918]
+- Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
 
 *Heartbeat*
 
 - Allow a list of status codes for HTTP checks. {pull}15587[15587]
-
+- Add additional ECS compatible fields for TLS information. {pull}17687[17687]
 
 *Journalbeat*
 
@@ -334,8 +351,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added documentation for running Metricbeat in Cloud Foundry. {pull}17275[17275]
 - Add test for documented fields check for metricsets without a http input. {issue}17315[17315] {pull}17334[17334]
 - Add final tests and move label to GA for the azure module in metricbeat. {pull}17319[17319]
+- Refactor windows/perfmon metricset configuration options and event output. {pull}17596[17596]
 - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429]
+- Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725]
 - Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609]
+- Allow partial region and zone name in googlecloud module config. {pull}17913[17913]
+- Add aggregation aligner as a config parameter for googlecloud stackdriver metricset. {issue}17141[[17141] {pull}17719[17719]
+- Move the perfmon metricset to GA. {issue}16608[16608] {pull}17879[17879]
+- Add static mapping for metricsets under aws module. {pull}17614[17614] {pull}17650[17650]
 - Update MSSQL module to fix some SSPI authentication and add brackets to USE statements {pull}17862[17862]]
 
 *Packetbeat*