network_traffic: remove beta tags on data streams and add flow tests (#5778)

efd6 · web-flow · commit 64cc1f8441e3 · 2023-04-06T10:21:28.000+09:30
* remove beta tags on data streams
* add flow is final filter for flows dashboard and flow tests
diff --git a/packages/network_traffic/_dev/build/docs/README.md b/packages/network_traffic/_dev/build/docs/README.md
@@ -141,6 +141,8 @@ The default value is 10s.
 
 {{fields "flow"}}
 
+{{event "flow"}}
+
 ## Protocols
 
 ### AMQP
diff --git a/packages/network_traffic/changelog.yml b/packages/network_traffic/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.11.0"
+  changes:
+    - description: Add flow is final filter to Network flow dashboard.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5778
+    - description: GA datastreams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5778
 - version: "1.10.1"
   changes:
     - description: Fix documentation for flows period.
diff --git a/packages/network_traffic/data_stream/amqp/manifest.yml b/packages/network_traffic/data_stream/amqp/manifest.yml
@@ -1,5 +1,4 @@
 title: AMQP
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/cassandra/manifest.yml b/packages/network_traffic/data_stream/cassandra/manifest.yml
@@ -1,5 +1,4 @@
 title: Cassandra
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/dhcpv4/manifest.yml b/packages/network_traffic/data_stream/dhcpv4/manifest.yml
@@ -1,5 +1,4 @@
 title: DHCP
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/dns/manifest.yml b/packages/network_traffic/data_stream/dns/manifest.yml
@@ -1,5 +1,4 @@
 title: DNS
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-http-get-2k-file-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-http-get-2k-file-config.yml
@@ -0,0 +1,6 @@
+vars:
+  interface: "{{SERVICE_LOGS_DIR}}/http_get_2k_file.pcap"
+input: packet
+data_stream:
+  vars:
+    flows.period: '-1s'
diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp-2-pings-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp-2-pings-config.yml
@@ -0,0 +1,6 @@
+vars:
+  interface: "{{SERVICE_LOGS_DIR}}/icmp_2_pings.pcap"
+input: packet
+data_stream:
+  vars:
+    flows.period: '-1s'
diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp4-ping-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp4-ping-config.yml
@@ -0,0 +1,6 @@
+vars:
+  interface: "{{SERVICE_LOGS_DIR}}/icmp4_ping.pcap"
+input: packet
+data_stream:
+  vars:
+    flows.period: '1s'
diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp6-ping-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp6-ping-config.yml
@@ -0,0 +1,6 @@
+vars:
+  interface: "{{SERVICE_LOGS_DIR}}/icmp6_ping.pcap"
+input: packet
+data_stream:
+  vars:
+    flows.period: '1s'
diff --git a/packages/network_traffic/data_stream/flow/elasticsearch/ingest_pipeline/default.yml b/packages/network_traffic/data_stream/flow/elasticsearch/ingest_pipeline/default.yml
@@ -5,7 +5,7 @@ processors:
     field: ecs.version
     value: '8.7.0'
 ##
-# Set host.mac to dash separated upper case value
+# Set {host,source,destination}.mac to dash separated upper case value
 # as per ECS recommendation
 ##
 - gsub:
@@ -21,6 +21,32 @@ processors:
 - uppercase:
     field: host.mac
     ignore_missing: true
+- gsub:
+    field: source.mac
+    pattern: '[-:.]'
+    replacement: ''
+    ignore_missing: true
+- gsub:
+    field: source.mac
+    pattern: '(..)(?!$)'
+    replacement: '$1-'
+    ignore_missing: true
+- uppercase:
+    field: source.mac
+    ignore_missing: true
+- gsub:
+    field: destination.mac
+    pattern: '[-:.]'
+    replacement: ''
+    ignore_missing: true
+- gsub:
+    field: destination.mac
+    pattern: '(..)(?!$)'
+    replacement: '$1-'
+    ignore_missing: true
+- uppercase:
+    field: destination.mac
+    ignore_missing: true
 
 - pipeline:
     if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich
diff --git a/packages/network_traffic/data_stream/flow/fields/ecs.yml b/packages/network_traffic/data_stream/flow/fields/ecs.yml
@@ -8,6 +8,10 @@
   name: destination.bytes
 - external: ecs
   name: destination.ip
+- external: ecs
+  name: destination.mac
+- external: ecs
+  name: destination.packets
 - external: ecs
   name: destination.port
 - external: ecs
@@ -36,6 +40,8 @@
   name: network.forwarded_ip
 - external: ecs
   name: network.protocol
+- external: ecs
+  name: network.packets
 - external: ecs
   name: network.transport
 - external: ecs
@@ -62,8 +68,12 @@
   name: source.bytes
 - external: ecs
   name: source.ip
+- external: ecs
+  name: source.mac
 - external: ecs
   name: source.port
+- external: ecs
+  name: source.packets
 - external: ecs
   name: client.geo.city_name
 - external: ecs
diff --git a/packages/network_traffic/data_stream/flow/manifest.yml b/packages/network_traffic/data_stream/flow/manifest.yml
@@ -1,5 +1,4 @@
 title: Flows
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/flow/sample_event.json b/packages/network_traffic/data_stream/flow/sample_event.json
@@ -0,0 +1,85 @@
+{
+    "@timestamp": "2023-04-04T23:12:40.755Z",
+    "agent": {
+        "ephemeral_id": "c368e835-b038-4610-b000-bc9fb23b35ab",
+        "id": "d35fb84a-73fb-4bc7-99b4-ac4df8c6ebb5",
+        "name": "docker-fleet-agent",
+        "type": "packetbeat",
+        "version": "8.6.2"
+    },
+    "data_stream": {
+        "dataset": "network_traffic.flow",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "destination": {
+        "bytes": 64,
+        "ip": "::1",
+        "packets": 1,
+        "port": 8000
+    },
+    "ecs": {
+        "version": "8.6.0"
+    },
+    "elastic_agent": {
+        "id": "d35fb84a-73fb-4bc7-99b4-ac4df8c6ebb5",
+        "snapshot": false,
+        "version": "8.6.2"
+    },
+    "event": {
+        "action": "network_flow",
+        "agent_id_status": "verified",
+        "category": [
+            "network"
+        ],
+        "dataset": "network_traffic.flow",
+        "duration": 70523,
+        "end": "2023-04-04T23:12:30.760Z",
+        "ingested": "2023-04-04T23:12:41Z",
+        "kind": "event",
+        "start": "2023-04-04T23:12:30.759Z",
+        "type": [
+            "connection"
+        ]
+    },
+    "flow": {
+        "final": false,
+        "id": "QAT///////8A////IP8AAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAUAfeMg"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "containerized": false,
+        "hostname": "docker-fleet-agent",
+        "id": "f91b175388d443fca5c155815dfc2279",
+        "ip": [
+            "192.168.208.7"
+        ],
+        "mac": [
+            "02-42-C0-A8-D0-07"
+        ],
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "5.15.49-linuxkit",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.5 LTS (Focal Fossa)"
+        }
+    },
+    "network": {
+        "bytes": 152,
+        "community_id": "1:5y9AkdbV9U8xqD9dhlj6obkubHg=",
+        "packets": 2,
+        "transport": "tcp",
+        "type": "ipv6"
+    },
+    "source": {
+        "bytes": 88,
+        "ip": "::1",
+        "packets": 1,
+        "port": 51320
+    },
+    "type": "flow"
+}
diff --git a/packages/network_traffic/data_stream/http/manifest.yml b/packages/network_traffic/data_stream/http/manifest.yml
@@ -1,5 +1,4 @@
 title: HTTP
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/icmp/manifest.yml b/packages/network_traffic/data_stream/icmp/manifest.yml
@@ -1,5 +1,4 @@
 title: ICMP
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/memcached/manifest.yml b/packages/network_traffic/data_stream/memcached/manifest.yml
@@ -1,5 +1,4 @@
 title: Memcached
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/mongodb/manifest.yml b/packages/network_traffic/data_stream/mongodb/manifest.yml
@@ -1,5 +1,4 @@
 title: MongoDB
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/mysql/manifest.yml b/packages/network_traffic/data_stream/mysql/manifest.yml
@@ -1,5 +1,4 @@
 title: MySQL
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/nfs/manifest.yml b/packages/network_traffic/data_stream/nfs/manifest.yml
@@ -1,5 +1,4 @@
 title: NFS
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/pgsql/manifest.yml b/packages/network_traffic/data_stream/pgsql/manifest.yml
@@ -1,5 +1,4 @@
 title: PostgreSQL
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/redis/manifest.yml b/packages/network_traffic/data_stream/redis/manifest.yml
@@ -1,5 +1,4 @@
 title: Redis
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/sip/manifest.yml b/packages/network_traffic/data_stream/sip/manifest.yml
@@ -1,5 +1,4 @@
 title: SIP
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/thrift/manifest.yml b/packages/network_traffic/data_stream/thrift/manifest.yml
@@ -1,5 +1,4 @@
 title: Thrift
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/data_stream/tls/manifest.yml b/packages/network_traffic/data_stream/tls/manifest.yml
@@ -1,5 +1,4 @@
 title: TLS
-release: beta
 type: logs
 streams:
   - input: packet
diff --git a/packages/network_traffic/docs/README.md b/packages/network_traffic/docs/README.md
diff --git a/packages/network_traffic/kibana/dashboard/network_traffic-flows.json b/packages/network_traffic/kibana/dashboard/network_traffic-flows.json
diff --git a/packages/network_traffic/manifest.yml b/packages/network_traffic/manifest.yml

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`title: AMQP`
`2`		`-release: beta`
`3`	`2`	`type: logs`
`4`	`3`	`streams:`
`5`	`4`	`- input: packet`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`title: Cassandra`
`2`		`-release: beta`
`3`	`2`	`type: logs`
`4`	`3`	`streams:`
`5`	`4`	`- input: packet`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`title: DHCP`
`2`		`-release: beta`
`3`	`2`	`type: logs`
`4`	`3`	`streams:`
`5`	`4`	`- input: packet`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`title: DNS`
`2`		`-release: beta`
`3`	`2`	`type: logs`
`4`	`3`	`streams:`
`5`	`4`	`- input: packet`