[Filebeat] Improve ECS categorization field mappings for zeek module (e…

…lastic#17738)

* Improve ECS categorization field mappings for zeek module

- capture_loss
  + convert pipeline to yaml
  + event.kind
  + event.type
- connection
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
- dce_rpc
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + related.ip
  + source.geo
  + destination.geo
- dhcp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
- dnp3
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + related.ip
  + source.geo
  + destination.geo
  + source.as
  + destiantion.as
- dns
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + dns.question.top_level_domain
  + related.ip
- dpd
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + source.geo
  + destiantion.geo
  + source.as
  + destiantion.as
  + related.ip
- files
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + file.mime_type
  + file.name
  + file.hash.md5
  + file.hash.sha1
  + file.hash.sha256
  + client.ip
  + server.ip
  + related.ip
  + related.hash
- ftp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + user.name
  + file.mime_type
  + file.size
  + related.ip
  + related.user
  + source.geo
  + destination.geo
- http
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + url.port type to number
  + http.request.method lowercase
  + related.ip
  + related.users
- intel
  + event.kind
  + event.type
  + related.ip
- irc
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + user.name
  + file.name
  + file.size
  + file.mime_type
  + related.ip
  + related.user
  + source.geo
  + destination.geo
- kerberos
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + user.name
  + user.domain
  + source.geo
  + destination.geo
  + related.ip
  + related.user
- modbus
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + related.ip
  + source.geo
  + destination.geo
- mysql
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + source.geo
  + destiantion.geo
  + related.ip
- notice
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + file.size
  + file.mime_type
  + rule.description
  + rule.name
- ntlm
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + user.name
  + user.domain
  + source.geo
  + destination.geo
  + related.ip
  + related.user
- ocsp
  + convert pipeline to yaml
  + event.kind
  + related.hash
- pe
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
- radius
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + user.name
  + source.geo
  + destination.geo
  + related.ip
  + related.user
- rdp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- rfb
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- sip
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + related.ip
  + source.geo
  + destination.geo
  + url.full
- smb_cmd
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + related.ip
  + related.user
  + source.geo
  + destination.geo
  + user.name
- smb_files
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + related.ip
  + related.user
  + source.geo
  + destination.geo
  + user.name
  + file.accessed
  + file.ctime
  + file.created
  + file.mtime
  + file.path
  + file.name
  + file.size
- smb_mapping
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- smtp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- snmp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- socks
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + related.ip
  + source.geo
  + destination.geo
  + user.name
  + related.user
- ssh
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + related.ip
  + source.geo
  + destination.geo
- ssl
  + event.kind
  + event.category
  + event.type
  + related.ip
- stats
  + convert pipeline to yaml
  + event.kind
- syslog
  + convert pipeline to yaml
  + event.kind
  + related.ip
  + source.geo
  + destination.geo
  + log.syslog.facility.name
  + log.syslog.severity.name
- traceroute
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- tunnel
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- weird
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
  + rule.name
- x509
  + event.kind
  + event.type

Closes elastic#16029

(cherry picked from commit b552dd8)

CHANGELOG.next.asciidoc

@@ -343,6 +343,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added Unix stream socket support as an input source and a syslog input source. {pull}17492[17492]
 - Improve ECS categorization field mappings in misp module. {issue}16026[16026] {pull}17344[17344]
 - Enhance `elasticsearch/deprecation` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17728[17728]
+- Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
 
 *Heartbeat*
 

x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml

@@ -0,0 +1,21 @@
+description: Pipeline for normalizing Zeek capture_loss.log
+processors:
+- set:
+    field: event.created
+    value: '{{_ingest.timestamp}}'
+- date:
+    field: zeek.capture_loss.ts
+    formats:
+    - UNIX
+- remove:
+    field: zeek.capture_loss.ts
+- set:
+    field: event.kind
+    value: metric
+- set:
+    field: event.type
+    value: info
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

x-pack/filebeat/module/zeek/capture_loss/manifest.yml

@@ -11,5 +11,5 @@ var:
   - name: tags
     default: [zeek.capture_loss]
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/capture_loss.yml

x-pack/filebeat/module/zeek/capture_loss/test/capture_loss-json.log-expected.json

@@ -2,7 +2,9 @@
     {
         "@timestamp": "2019-09-10T16:19:28.465Z",
         "event.dataset": "zeek.capture_loss",
+        "event.kind": "metric",
         "event.module": "zeek",
+        "event.type": "info",
         "fileset.name": "capture_loss",
         "input.type": "log",
         "log.offset": 0,

x-pack/filebeat/module/zeek/connection/config/connection.yml

@@ -75,20 +75,27 @@ processors:
 
       ignore_missing: true
       fail_on_error: false
-
+  - convert:
+      fields:
+        - {from: "zeek.session_id", to: "event.id"}
+        - {from: "source.address", to: "source.ip", type: "ip"}
+        - {from: "destination.address", to: "destination.ip", type: "ip"}
+      ignore_missing: true
+      fail_on_error: false
+  - add_fields:
+      target: event
+      fields:
+        kind: event
+        category:
+          - network
 {{ if .community_id }}
   - if:
       equals.network.transport: icmp
     then:
       community_id:
         fields:
-          source_ip: source.address
-          destination_ip: destination.address
           icmp_type: zeek.connection.icmp.type
           icmp_code: zeek.connection.icmp.code
     else:
       community_id:
-        fields:
-          source_ip: source.address
-          destination_ip: destination.address
 {{ end }}