Add detailed docker network summary stats (elastic#25354)

* add detailed network summary stats to docker/memory

* add changelog

* update deps, notice

* update ref yml

* fix test

* move to new metricset, refactor

* revert changes to docker/network

* remove config file

* update xpack

* small fixes

CHANGELOG.next.asciidoc

@@ -979,6 +979,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022]
 - Release MSSQL as GA {pull}23146[23146]
 - Add support for SASL/SCRAM authentication to the Kafka module. {pull}24810[24810]
+- Add additional network metrics to docker/network {pull}25354[25354]
 
 *Packetbeat*
 

NOTICE.txt

@@ -7695,11 +7695,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.6.0
+Version: v1.7.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.6.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.7.0/LICENSE.txt:
 
 
                                  Apache License

go.mod

@@ -70,7 +70,7 @@ require (
 	github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
 	github.com/elastic/go-seccomp-bpf v1.1.0
 	github.com/elastic/go-structform v0.0.8
-	github.com/elastic/go-sysinfo v1.6.0
+	github.com/elastic/go-sysinfo v1.7.0
 	github.com/elastic/go-txfile v0.0.7
 	github.com/elastic/go-ucfg v0.8.3
 	github.com/elastic/go-windows v1.0.1 // indirect

go.sum

@@ -299,8 +299,8 @@ github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE
 github.com/elastic/go-structform v0.0.8 h1:U0qnb9Zqig7w+FhF+sLI3VZPPi/+2aJ0bIEW6R1z6Tk=
 github.com/elastic/go-structform v0.0.8/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
 github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
-github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw=
-github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
+github.com/elastic/go-sysinfo v1.7.0 h1:4vVvcfi255+8+TyQ7TYUTEK3A+G8v5FLE+ZKYL1z1Dg=
+github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
 github.com/elastic/go-txfile v0.0.7 h1:Yn28gclW7X0Qy09nSMSsx0uOAvAGMsp6XHydbiLVe2s=
 github.com/elastic/go-txfile v0.0.7/go.mod h1:H0nCoFae0a4ga57apgxFsgmRjevNCsEaT6g56JoeKAE=
 github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
@@ -765,11 +765,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b h1:X/8hkb4rQq3+QuOxpJK7gWmAXmZucF0EI1s1BfBLq6U=
 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC1PwcLTWd6rjQyGyItxxrhpiBl8LSuAGmw=

metricbeat/module/system/network_summary/data.go → libbeat/metric/system/network/helpers.go

@@ -15,15 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package network_summary
+package network
 
 import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	sysinfotypes "github.com/elastic/go-sysinfo/types"
 )
 
-// eventMapping maps the network counters to a MapStr that wil be sent to report.Event
-func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr {
+// MapProcNetCounters converts the NetworkCountersInfo struct into a MapStr acceptable for sending upstream
+func MapProcNetCounters(raw *sysinfotypes.NetworkCountersInfo) common.MapStr {
 
 	eventByProto := common.MapStr{
 		"ip":       combineMap(raw.Netstat.IPExt, raw.SNMP.IP),
@@ -33,10 +33,6 @@ func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr {
 		"icmp":     combineMap(raw.SNMP.ICMPMsg, raw.SNMP.ICMP),
 	}
 
-	// if value, ok := raw.SNMP.TCP["MaxConn"]; ok && value != 0 {
-	// 	eventByProto["tcp"].(map)
-	// }
-
 	return eventByProto
 }
 

metricbeat/docs/fields.asciidoc

@@ -12386,6 +12386,83 @@ type: long
 Total number of outgoing packets.
 
 
+type: long
+
+--
+
+[float]
+=== network_summary
+
+network_summary
+
+
+
+*`docker.network_summary.ip.*`*::
++
+--
+IP counters
+
+
+type: object
+
+--
+
+*`docker.network_summary.tcp.*`*::
++
+--
+TCP counters
+
+
+type: object
+
+--
+
+*`docker.network_summary.udp.*`*::
++
+--
+UDP counters
+
+
+type: object
+
+--
+
+*`docker.network_summary.udp_lite.*`*::
++
+--
+UDP Lite counters
+
+
+type: object
+
+--
+
+*`docker.network_summary.icmp.*`*::
++
+--
+ICMP counters
+
+
+type: object
+
+--
+
+*`docker.network_summary.namespace.pid`*::
++
+--
+The root PID of the container, corresponding to /proc/[pid]/net
+
+
+type: long
+
+--
+
+*`docker.network_summary.namespace.id`*::
++
+--
+The ID of the network namespace used by the container, corresponding to /proc/[pid]/ns/net
+
+
 type: long
 
 --

metricbeat/docs/modules/docker.asciidoc

@@ -44,6 +44,7 @@ metricbeat.modules:
     #- "image"
     - "memory"
     - "network"
+    #- "network_summary"
   hosts: ["unix:///var/run/docker.sock"]
   period: 10s
   enabled: true
@@ -86,6 +87,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-docker-network,network>>
 
+* <<metricbeat-metricset-docker-network_summary,network_summary>>
+
 include::docker/container.asciidoc[]
 
 include::docker/cpu.asciidoc[]
@@ -104,3 +107,5 @@ include::docker/memory.asciidoc[]
 
 include::docker/network.asciidoc[]
 
+include::docker/network_summary.asciidoc[]
+

metricbeat/docs/modules/docker/network_summary.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-docker-network_summary]]
+=== Docker network_summary metricset
+
+beta[]
+
+include::../../../module/docker/network_summary/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-docker,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/docker/network_summary/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -80,7 +80,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-module-couchdb,CouchDB>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-couchdb-server,server>>   
 |<<metricbeat-module-docker,Docker>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.9+| .9+|  |<<metricbeat-metricset-docker-container,container>>   
+.10+| .10+|  |<<metricbeat-metricset-docker-container,container>>   
 |<<metricbeat-metricset-docker-cpu,cpu>>   
 |<<metricbeat-metricset-docker-diskio,diskio>>   
 |<<metricbeat-metricset-docker-event,event>>   
@@ -89,6 +89,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-docker-info,info>>   
 |<<metricbeat-metricset-docker-memory,memory>>   
 |<<metricbeat-metricset-docker-network,network>>   
+|<<metricbeat-metricset-docker-network_summary,network_summary>> beta[]  
 |<<metricbeat-module-dropwizard,Dropwizard>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-dropwizard-collector,collector>>   
 |<<metricbeat-module-elasticsearch,Elasticsearch>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  

metricbeat/metricbeat.reference.yml

@@ -235,6 +235,7 @@ metricbeat.modules:
     #- "image"
     - "memory"
     - "network"
+    #- "network_summary"
   hosts: ["unix:///var/run/docker.sock"]
   period: 10s
   enabled: true

metricbeat/module/docker/_meta/config.reference.yml

@@ -9,6 +9,7 @@
     #- "image"
     - "memory"
     - "network"
+    #- "network_summary"
   hosts: ["unix:///var/run/docker.sock"]
   period: 10s
   enabled: true

metricbeat/module/docker/_meta/config.yml

@@ -8,6 +8,7 @@
   #  - info
   #  - memory
   #  - network
+  #  - network_summary
   period: 10s
   hosts: ["unix:///var/run/docker.sock"]
 

metricbeat/module/docker/network/_meta/data.json

@@ -1,36 +1,48 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "container": {
-        "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a",
+        "id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80",
         "image": {
-            "name": "metricbeat_elasticsearch"
+            "name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT"
         },
-        "name": "metricbeat_elasticsearch_1_df866b3a7b3d",
+        "name": "elastic-package-stack_elastic-agent_1",
         "runtime": "docker"
     },
     "docker": {
         "container": {
             "labels": {
-                "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78",
+                "com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3",
                 "com_docker_compose_container-number": "1",
                 "com_docker_compose_oneoff": "False",
-                "com_docker_compose_project": "metricbeat",
-                "com_docker_compose_service": "elasticsearch",
-                "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d",
-                "com_docker_compose_version": "1.23.1",
+                "com_docker_compose_project": "elastic-package-stack",
+                "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml",
+                "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport",
+                "com_docker_compose_service": "elastic-agent",
+                "com_docker_compose_version": "1.27.4",
+                "description": "Agent manages other beats based on configuration provided.",
+                "io_k8s_description": "Agent manages other beats based on configuration provided.",
+                "io_k8s_display-name": "Elastic-Agent image",
                 "license": "Elastic License",
-                "org_label-schema_build-date": "20181006",
-                "org_label-schema_license": "GPLv2",
-                "org_label-schema_name": "elasticsearch",
+                "maintainer": "infra@elastic.co",
+                "name": "elastic-agent",
+                "org_label-schema_build-date": "2021-04-21T06:45:43Z",
+                "org_label-schema_license": "Elastic License",
+                "org_label-schema_name": "elastic-agent",
                 "org_label-schema_schema-version": "1.0",
-                "org_label-schema_url": "https://www.elastic.co/products/elasticsearch",
-                "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker",
+                "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent",
+                "org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf",
+                "org_label-schema_vcs-url": "github.com/elastic/beats/v7",
                 "org_label-schema_vendor": "Elastic",
-                "org_label-schema_version": "6.5.1"
+                "org_label-schema_version": "7.13.0-SNAPSHOT",
+                "org_opencontainers_image_created": "2021-04-21T06:45:43Z",
+                "org_opencontainers_image_licenses": "Elastic License",
+                "org_opencontainers_image_title": "Elastic-Agent",
+                "org_opencontainers_image_vendor": "Elastic",
+                "release": "1",
+                "summary": "elastic-agent",
+                "url": "https://www.elastic.co/beats/elastic-agent",
+                "vendor": "Elastic",
+                "version": "7.13.0-SNAPSHOT"
             }
         },
         "network": {
@@ -41,10 +53,10 @@
                 "packets": 0
             },
             "inbound": {
-                "bytes": 23047,
+                "bytes": 156469392,
                 "dropped": 0,
                 "errors": 0,
-                "packets": 241
+                "packets": 710866
             },
             "interface": "eth0",
             "out": {
@@ -54,10 +66,10 @@
                 "packets": 0
             },
             "outbound": {
-                "bytes": 0,
+                "bytes": 3150916813,
                 "dropped": 0,
                 "errors": 0,
-                "packets": 0
+                "packets": 1059447
             }
         }
     },
@@ -67,7 +79,8 @@
         "module": "docker"
     },
     "metricset": {
-        "name": "network"
+        "name": "network",
+        "period": 10000
     },
     "service": {
         "address": "/var/run/docker.sock",