Merge branch 'master' into fix-700

CHANGELOG-developer.asciidoc

@@ -12,6 +12,26 @@ other Beats should be migrated.
 
 Note: This changelog was only started after the 6.3 release.
 
+=== Beats version 7.0.0-rc2
+https://github.com/elastic/beats/compare/v7.0.0-rc1..v7.0.0-rc2[Check the HEAD diff]
+
+=== Beats version 7.0.0-rc1
+https://github.com/elastic/beats/compare/v7.0.0-beta1..v7.0.0-rc1[Check the HEAD diff]
+
+==== Breaking changes
+
+- Remove support for deprecated `GenRootCmd` methods. {pull}10721[10721]
+- Remove SkipNormalization, SkipAgentMetadata, SkipAddHostName. {pull}10801[10801] {pull}10769[10769]
+
+==== Bugfixes
+
+- Align default index between elasticsearch and logstash and kafka output. {pull}10841[10841]
+- Fix duplication check for `append_fields` option. {pull}10959[10959]
+
+==== Added
+
+- Introduce processing.Support to instance.Setting. This allows Beats to fully modify the event processing. {pull}10801[10801]
+
 === Beats version 7.0.0-beta1
 https://github.com/elastic/beats/compare/v7.0.0-alpha2..v7.0.0-beta1[Check the HEAD diff]
 

CHANGELOG-developer.next.asciidoc

@@ -14,28 +14,22 @@ Note: This changelog documents the current changes which are not yet present in
 an actual release.
 
 === Beats version HEAD
-https://github.com/elastic/beats/compare/v7.0.0-beta1..master[Check the HEAD diff]
+https://github.com/elastic/beats/compare/v7.0.0-rc2..master[Check the HEAD diff]
 
-The list below covers the major changes between 7.0.0-beta1 and master only.
+The list below covers the major changes between 7.0.0-rc2 and master only.
 
 ==== Breaking changes
 
-- Remove support for deprecated `GenRootCmd` methods. {pull}10721[10721]
-- Remove SkipNormalization, SkipAgentMetadata, SkipAddHostName. {pull}10801[10801] {pull}10769[10769]
 - Move Fields from package libbeat/common to libbeat/mapping. {pull}11198[11198]
 
 ==== Bugfixes
 
-- Align default index between elasticsearch and logstash and kafka output. {pull}10841[10841]
-- Fix duplication check for `append_fields` option. {pull}10959[10959]
-
 ==== Added
 
 - Metricset generator generates beta modules by default now. {pull}10657[10657]
 - The `beat.Event` accessor methods now support `@metadata` keys. {pull}10761[10761]
 - Assertion for documented fields in tests fails if any of the fields in the tested event is documented as an alias. {pull}10921[10921]
 - Support for Logger in the Metricset base instance. {pull}11106[11106]
-- Introduce processing.Support to instance.Setting. This allows Beats to fully modify the event processing. {pull}10801[10801]
 - Filebeat modules can now use ingest pipelines in YAML format. {pull}11209[11209]
 - Added support for using PYTHON_EXE to control what Python interpreter is used
   by `make` and `mage`. Example: `export PYTHON_EXE=python2.7`. {pull}11212[11212]

Vagrantfile

@@ -69,17 +69,19 @@ if [ -d "/vagrant" ]  && [ ! -e "beats" ]; then ln -s /vagrant beats; fi
 SCRIPT
 
 # Linux GVM
-$linuxGvmProvision = <<SCRIPT
+def linuxGvmProvision(arch="amd64")
+  return <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-amd64
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-#{arch}
   chmod +x ~/bin/gvm
-  ~/bin/gvm $GO_VERSION
+  ~/bin/gvm #{GO_VERSION}
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
   echo 'export PATH=$HOME/bin:$GOPATH/bin:$PATH' >> ~/.bash_profile
   echo 'eval "$(gvm #{GO_VERSION})"' >> ~/.bash_profile
 fi
 SCRIPT
+end
 
 Vagrant.configure(2) do |config|
 
@@ -141,7 +143,17 @@ Vagrant.configure(2) do |config|
     c.vm.network :forwarded_port, guest: 22,   host: 2226,  id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
+
+    c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
+  end
+
+  config.vm.define "precise32", primary: true do |c|
+    c.vm.box = "ubuntu/precise32"
+    c.vm.network :forwarded_port, guest: 22,   host: 2226,  id: "ssh", auto_correct: true
+
+    c.vm.provision "shell", inline: $unixProvision, privileged: false
+    c.vm.provision "shell", inline: linuxGvmProvision("386"), privileged: false
 
     c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
   end
@@ -151,7 +163,7 @@ Vagrant.configure(2) do |config|
     c.vm.network :forwarded_port, guest: 22,   host: 2229,  id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
     c.vm.provision "shell", inline: "yum install -y make gcc python-pip python-virtualenv git"
 
     c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
@@ -162,7 +174,7 @@ Vagrant.configure(2) do |config|
     c.vm.network :forwarded_port, guest: 22,   host: 2227,  id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
     c.vm.provision "shell", inline: "dnf install -y make gcc python-pip python-virtualenv git"
 
     c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
@@ -173,7 +185,7 @@ Vagrant.configure(2) do |config|
     c.vm.network :forwarded_port, guest: 22,   host: 2228,  id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
     c.vm.provision "shell", inline: "pacman -Sy && pacman -S --noconfirm make gcc python-pip python-virtualenv git"
 
     c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
@@ -184,7 +196,7 @@ Vagrant.configure(2) do |config|
     c.vm.network :forwarded_port, guest: 22,   host: 2229,  id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
     c.vm.provision "shell", inline: "apt-get update && apt-get install -y make gcc python-pip python-virtualenv git"
 
     c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
@@ -195,7 +207,7 @@ Vagrant.configure(2) do |config|
     c.vm.network :forwarded_port, guest: 22,   host: 2230,  id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
     c.vm.provision "shell", inline: "pip install virtualenv"
 
     c.vm.synced_folder ".", "/vagrant", type: "virtualbox"

auditbeat/auditbeat.reference.yml

@@ -339,6 +339,32 @@ auditbeat.modules:
 #          to: message_copied
 #    fail_on_error: true
 #    ignore_missing: false
+#
+# The following example truncates the value of message to 1024 bytes
+#
+#processors:
+#- truncate_fields:
+#    fields:
+#      - message
+#    max_bytes: 1024
+#    fail_on_error: false
+#    ignore_missing: true
+#
+# The following example preserves the raw message under event.original
+#
+#processors:
+#- copy_fields:
+#    fields:
+#        - from: message
+#          to: event.original
+#    fail_on_error: false
+#    ignore_missing: true
+#- truncate_fields:
+#    fields:
+#      - event.original
+#    max_bytes: 1024
+#    fail_on_error: false
+#    ignore_missing: true
 
 #============================= Elastic Cloud ==================================
 

auditbeat/auditbeat.yml

@@ -178,4 +178,4 @@ processors:
 #================================= Migration ==================================
 
 # This allows to enable 6.7 migration aliases
-#migration.6_to_7.enabled: false
+#migration.6_to_7.enabled: true

auditbeat/docs/breaking.asciidoc

@@ -1,49 +1,4 @@
 [[auditbeat-breaking-changes]]
-== Breaking changes in 7.0
-
-In version 7.0 the following fields were renamed.
-
-.Renamed Fields in 7.0
-[frame="topbot",options="header"]
-|======================
-|Old Field|New Field
-|`auditd.messages`        |`event.original`
-|`auditd.warnings`        |`error.message`
-|`event.type`             |`auditd.message_type`
-|`process.cwd`            |`process.working_directory`
-|`source.hostname`        |`source.domain`
-|`user.auid`              |`user.audit.id`
-|`user.uid`               |`user.id`
-|`user.euid`              |`user.effective.id`
-|`user.fsuid`             |`user.filesystem.id`
-|`user.suid`              |`user.saved.id`
-|`user.gid`               |`user.group.id`
-|`user.egid`              |`user.effective.group.id`
-|`user.sgid`              |`user.saved.group.id`
-|`user.fsgid`             |`user.filesystem.group.id`
-|`user.name_map.auid`     |`user.audit.name`
-|`user.name_map.uid`      |`user.name`
-|`user.name_map.euid`     |`user.effective.name`
-|`user.name_map.fsuid`    |`user.filesystem.name`
-|`user.name_map.suid`     |`user.saved.name`
-|`user.name_map.gid`      |`user.group.name`
-|`user.name_map.egid`     |`user.effective.group.name`
-|`user.name_map.sgid`     |`user.saved.group.name`
-|`user.name_map.fsgid`    |`user.filesystem.group.name`
-|======================
-
-The JSON data types produced by the output have been changed to align with
-the data types used in the Elasticsearch index template.
-
-.Type Changes in 7.0
-[frame="topbot",options="header"]
-|======================
-|Field|Old Type|New Type
-|`file.gid`     |number |string
-|`file.uid`     |number |string
-|`process.pid`  |string |number
-|`process.ppid` |string |number
-|======================
 
 == Breaking changes in 6.2
 

auditbeat/module/auditd/audit_linux.go

@@ -467,12 +467,17 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 		aucoalesce.ResolveIDs(auditEvent)
 	}
 
+	eventOutcome := auditEvent.Result
+	if eventOutcome == "fail" {
+		eventOutcome = "failure"
+	}
 	out := mb.Event{
 		Timestamp: auditEvent.Timestamp,
 		RootFields: common.MapStr{
 			"event": common.MapStr{
 				"category": auditEvent.Category.String(),
 				"action":   auditEvent.Summary.Action,
+				"outcome":  eventOutcome,
 			},
 		},
 		ModuleFields: common.MapStr{
@@ -484,6 +489,9 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 		},
 	}
 
+	// Customize event.type / event.category to match unified values.
+	normalizeEventFields(out.RootFields)
+
 	// Add root level fields.
 	addUser(auditEvent.User, out.RootFields)
 	addProcess(auditEvent.Process, out.RootFields)
@@ -533,6 +541,28 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 	return out
 }
 
+func normalizeEventFields(m common.MapStr) {
+	getFieldAsStr := func(key string) (s string, found bool) {
+		iface, err := m.GetValue(key)
+		if err != nil {
+			return
+		}
+		s, found = iface.(string)
+		return
+	}
+
+	category, ok1 := getFieldAsStr("event.category")
+	action, ok2 := getFieldAsStr("event.action")
+	outcome, ok3 := getFieldAsStr("event.outcome")
+	if !ok1 || !ok2 || !ok3 {
+		return
+	}
+	if category == "user-login" && action == "logged-in" { // USER_LOGIN
+		m.Put("event.category", "authentication")
+		m.Put("event.type", fmt.Sprintf("authentication_%s", outcome))
+	}
+}
+
 func addUser(u aucoalesce.User, m common.MapStr) {
 	user := common.MapStr{}
 	m.Put("user", user)

auditbeat/module/auditd/audit_linux_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/prometheus/procfs"
 
 	"github.com/elastic/beats/auditbeat/core"
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/mapping"
 	"github.com/elastic/beats/metricbeat/mb"
@@ -48,7 +49,8 @@ import (
 var audit = flag.Bool("audit", false, "interact with the real audit framework")
 
 var (
-	userLoginMsg = `type=USER_LOGIN msg=audit(1492896301.818:19955): pid=12635 uid=0 auid=4294967295 ses=4294967295 msg='op=login acct=28696E76616C6964207573657229 exe="/usr/sbin/sshd" hostname=? addr=179.38.151.221 terminal=sshd res=failed'`
+	userLoginFailMsg    = `type=USER_LOGIN msg=audit(1492896301.818:19955): pid=12635 uid=0 auid=4294967295 ses=4294967295 msg='op=login acct=28696E76616C6964207573657229 exe="/usr/sbin/sshd" hostname=? addr=179.38.151.221 terminal=sshd res=failed'`
+	userLoginSuccessMsg = `type=USER_LOGIN msg=audit(1492896303.915:19956): pid=12635 uid=0 auid=4294967295 ses=4294967295 msg='op=login acct=28696E76616C6964207573657229 exe="/usr/sbin/sshd" hostname=? addr=179.38.151.221 terminal=sshd res=success'`
 
 	execveMsgs = []string{
 		`type=SYSCALL msg=audit(1492752522.985:8972): arch=c000003e syscall=59 success=yes exit=0 a0=10812c8 a1=1070208 a2=1152008 a3=59a items=2 ppid=10027 pid=10043 auid=1001 uid=1001 gid=1002 euid=1001 suid=1001 fsuid=1001 egid=1002 sgid=1002 fsgid=1002 tty=pts0 ses=11 comm="uname" exe="/bin/uname" key="key=user_commands"`,
@@ -77,8 +79,8 @@ func TestData(t *testing.T) {
 		returnACK().returnStatus().
 		// Send expected ACKs for initialization
 		returnACK().returnACK().returnACK().returnACK().returnACK().
-		// Send a single audit message from the kernel.
-		returnMessage(userLoginMsg).
+		// Send three auditd messages.
+		returnMessage(userLoginFailMsg).
 		returnMessage(execveMsgs...).
 		returnMessage(acceptMsgs...)
 
@@ -100,6 +102,61 @@ func TestData(t *testing.T) {
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }
 
+func TestLoginType(t *testing.T) {
+	logp.TestingSetup()
+
+	// Create a mock netlink client that provides the expected responses.
+	mock := NewMock().
+		// Get Status response for initClient
+		returnACK().returnStatus().
+		// Send expected ACKs for initialization
+		returnACK().returnACK().returnACK().returnACK().returnACK().
+		// Send an authentication failure and a success.
+		returnMessage(userLoginFailMsg).
+		returnMessage(userLoginSuccessMsg)
+
+	// Replace the default AuditClient with a mock.
+	ms := mbtest.NewPushMetricSetV2(t, getConfig())
+	auditMetricSet := ms.(*MetricSet)
+	auditMetricSet.client.Close()
+	auditMetricSet.client = &libaudit.AuditClient{Netlink: mock}
+
+	events := mbtest.RunPushMetricSetV2(10*time.Second, 2, ms)
+	if len(events) != 2 {
+		t.Fatalf("expected 2 events, but received %d", len(events))
+	}
+	assertNoErrors(t, events)
+
+	assertFieldsAreDocumented(t, events)
+
+	// Sometimes the events are received in reverse order.
+	if events[0].ModuleFields["sequence"].(uint32) > events[1].ModuleFields["sequence"].(uint32) {
+		events[0], events[1] = events[1], events[0]
+	}
+
+	for idx, expected := range []common.MapStr{
+		{
+			"event.category": "authentication",
+			"event.type":     "authentication_failure",
+			"event.outcome":  "failure",
+		},
+		{
+			"event.category": "authentication",
+			"event.type":     "authentication_success",
+			"event.outcome":  "success",
+		},
+	} {
+		beatEvent := mbtest.StandardizeEvent(ms, events[idx], core.AddDatasetToEvent)
+		mbtest.WriteEventToDataJSON(t, beatEvent, "")
+		for k, v := range expected {
+			msg := fmt.Sprintf("%s[%d]", k, idx)
+			cur, err := beatEvent.GetValue(k)
+			assert.NoError(t, err, msg)
+			assert.Equal(t, v, cur, msg)
+		}
+	}
+}
+
 // assertFieldsAreDocumented mimics assert_fields_are_documented in Python system tests.
 func assertFieldsAreDocumented(t *testing.T, events []mb.Event) {
 	fieldsYml, err := mapping.LoadFieldsYaml("../../fields.yml")

filebeat/docs/modules/coredns.asciidoc

@@ -11,8 +11,7 @@ This file is generated! See scripts/docs_collector.py
 == Coredns Module
 
 This is a filebeat module for coredns. It supports both standalone coredns deployment and 
-coredns deployment in Kubernetes. Standalone coredns deployment uses the log fileset, while
-Kubernetes coredns deployment uses the kubernetes fileset.
+coredns deployment in Kubernetes. 
 
 [float]
 === Compatibility