From f7e80da3c6292fe64ef93866feb2ed0315bab1e5 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Thu, 6 May 2021 09:33:19 -0700 Subject: [PATCH] Add detailed docker network summary stats (#25354) * add detailed network summary stats to docker/memory * add changelog * update deps, notice * update ref yml * fix test * move to new metricset, refactor * revert changes to docker/network * remove config file * update xpack * small fixes --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 4 +- go.mod | 2 +- go.sum | 7 +- .../metric/system/network/helpers.go | 10 +- metricbeat/docs/fields.asciidoc | 77 +++++ metricbeat/docs/modules/docker.asciidoc | 5 + .../modules/docker/network_summary.asciidoc | 23 ++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list_docker.go | 1 + metricbeat/metricbeat.reference.yml | 1 + .../module/docker/_meta/config.reference.yml | 1 + metricbeat/module/docker/_meta/config.yml | 1 + metricbeat/module/docker/fields.go | 2 +- .../module/docker/network/_meta/data.json | 59 ++-- .../docker/network_summary/_meta/data.json | 296 ++++++++++++++++++ .../network_summary/_meta/docs.asciidoc | 4 + .../docker/network_summary/_meta/fields.yml | 34 ++ .../module/docker/network_summary/config.go | 37 +++ .../module/docker/network_summary/doc.go | 18 ++ .../module/docker/network_summary/helper.go | 94 ++++++ .../docker/network_summary/network_summary.go | 129 ++++++++ .../network_summary/network_summary_test.go | 39 +++ .../system/network_summary/network_summary.go | 3 +- .../network_summary/network_summary_test.go | 3 +- metricbeat/modules.d/docker.yml.disabled | 1 + x-pack/metricbeat/metricbeat.reference.yml | 1 + 27 files changed, 815 insertions(+), 41 deletions(-) rename metricbeat/module/system/network_summary/data.go => libbeat/metric/system/network/helpers.go (86%) create mode 100644 metricbeat/docs/modules/docker/network_summary.asciidoc create mode 100644 metricbeat/module/docker/network_summary/_meta/data.json create mode 100644 metricbeat/module/docker/network_summary/_meta/docs.asciidoc create mode 100644 metricbeat/module/docker/network_summary/_meta/fields.yml create mode 100644 metricbeat/module/docker/network_summary/config.go create mode 100644 metricbeat/module/docker/network_summary/doc.go create mode 100644 metricbeat/module/docker/network_summary/helper.go create mode 100644 metricbeat/module/docker/network_summary/network_summary.go create mode 100644 metricbeat/module/docker/network_summary/network_summary_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ca1cd95cba97..753979acc30b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -979,6 +979,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022] - Release MSSQL as GA {pull}23146[23146] - Add support for SASL/SCRAM authentication to the Kafka module. {pull}24810[24810] +- Add additional network metrics to docker/network {pull}25354[25354] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index bbbc50ec0d94..f9893b43dddd 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -7695,11 +7695,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-sysinfo -Version: v1.6.0 +Version: v1.7.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.6.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.7.0/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 0b2bcc9c991d..47278a56e8be 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 github.com/elastic/go-seccomp-bpf v1.1.0 github.com/elastic/go-structform v0.0.8 - github.com/elastic/go-sysinfo v1.6.0 + github.com/elastic/go-sysinfo v1.7.0 github.com/elastic/go-txfile v0.0.7 github.com/elastic/go-ucfg v0.8.3 github.com/elastic/go-windows v1.0.1 // indirect diff --git a/go.sum b/go.sum index ee234fb91d85..67010c699485 100644 --- a/go.sum +++ b/go.sum @@ -299,8 +299,8 @@ github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE github.com/elastic/go-structform v0.0.8 h1:U0qnb9Zqig7w+FhF+sLI3VZPPi/+2aJ0bIEW6R1z6Tk= github.com/elastic/go-structform v0.0.8/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= -github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw= -github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-sysinfo v1.7.0 h1:4vVvcfi255+8+TyQ7TYUTEK3A+G8v5FLE+ZKYL1z1Dg= +github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-txfile v0.0.7 h1:Yn28gclW7X0Qy09nSMSsx0uOAvAGMsp6XHydbiLVe2s= github.com/elastic/go-txfile v0.0.7/go.mod h1:H0nCoFae0a4ga57apgxFsgmRjevNCsEaT6g56JoeKAE= github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= @@ -765,11 +765,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b h1:X/8hkb4rQq3+QuOxpJK7gWmAXmZucF0EI1s1BfBLq6U= github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC1PwcLTWd6rjQyGyItxxrhpiBl8LSuAGmw= diff --git a/metricbeat/module/system/network_summary/data.go b/libbeat/metric/system/network/helpers.go similarity index 86% rename from metricbeat/module/system/network_summary/data.go rename to libbeat/metric/system/network/helpers.go index 95b530c93252..e13ad0e2ad70 100644 --- a/metricbeat/module/system/network_summary/data.go +++ b/libbeat/metric/system/network/helpers.go @@ -15,15 +15,15 @@ // specific language governing permissions and limitations // under the License. -package network_summary +package network import ( "github.com/elastic/beats/v7/libbeat/common" sysinfotypes "github.com/elastic/go-sysinfo/types" ) -// eventMapping maps the network counters to a MapStr that wil be sent to report.Event -func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr { +// MapProcNetCounters converts the NetworkCountersInfo struct into a MapStr acceptable for sending upstream +func MapProcNetCounters(raw *sysinfotypes.NetworkCountersInfo) common.MapStr { eventByProto := common.MapStr{ "ip": combineMap(raw.Netstat.IPExt, raw.SNMP.IP), @@ -33,10 +33,6 @@ func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr { "icmp": combineMap(raw.SNMP.ICMPMsg, raw.SNMP.ICMP), } - // if value, ok := raw.SNMP.TCP["MaxConn"]; ok && value != 0 { - // eventByProto["tcp"].(map) - // } - return eventByProto } diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 50ea29c3e9b9..1f12e70e1a0b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -12386,6 +12386,83 @@ type: long Total number of outgoing packets. +type: long + +-- + +[float] +=== network_summary + +network_summary + + + +*`docker.network_summary.ip.*`*:: ++ +-- +IP counters + + +type: object + +-- + +*`docker.network_summary.tcp.*`*:: ++ +-- +TCP counters + + +type: object + +-- + +*`docker.network_summary.udp.*`*:: ++ +-- +UDP counters + + +type: object + +-- + +*`docker.network_summary.udp_lite.*`*:: ++ +-- +UDP Lite counters + + +type: object + +-- + +*`docker.network_summary.icmp.*`*:: ++ +-- +ICMP counters + + +type: object + +-- + +*`docker.network_summary.namespace.pid`*:: ++ +-- +The root PID of the container, corresponding to /proc/[pid]/net + + +type: long + +-- + +*`docker.network_summary.namespace.id`*:: ++ +-- +The ID of the network namespace used by the container, corresponding to /proc/[pid]/ns/net + + type: long -- diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc index 56783db38689..bafd02ff5d58 100644 --- a/metricbeat/docs/modules/docker.asciidoc +++ b/metricbeat/docs/modules/docker.asciidoc @@ -44,6 +44,7 @@ metricbeat.modules: #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true @@ -86,6 +87,8 @@ The following metricsets are available: * <> +* <> + include::docker/container.asciidoc[] include::docker/cpu.asciidoc[] @@ -104,3 +107,5 @@ include::docker/memory.asciidoc[] include::docker/network.asciidoc[] +include::docker/network_summary.asciidoc[] + diff --git a/metricbeat/docs/modules/docker/network_summary.asciidoc b/metricbeat/docs/modules/docker/network_summary.asciidoc new file mode 100644 index 000000000000..3d0510ac68f0 --- /dev/null +++ b/metricbeat/docs/modules/docker/network_summary.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-network_summary]] +=== Docker network_summary metricset + +beta[] + +include::../../../module/docker/network_summary/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/network_summary/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 688f9235d076..cd1e738c9e43 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -80,7 +80,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.9+| .9+| |<> +.10+| .10+| |<> |<> |<> |<> @@ -89,6 +89,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go index 6a9201efc723..2e6e65d84fce 100644 --- a/metricbeat/include/list_docker.go +++ b/metricbeat/include/list_docker.go @@ -32,6 +32,7 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/docker/info" _ "github.com/elastic/beats/v7/metricbeat/module/docker/memory" _ "github.com/elastic/beats/v7/metricbeat/module/docker/network" + _ "github.com/elastic/beats/v7/metricbeat/module/docker/network_summary" _ "github.com/elastic/beats/v7/metricbeat/module/kubernetes/apiserver" _ "github.com/elastic/beats/v7/metricbeat/module/kubernetes/container" _ "github.com/elastic/beats/v7/metricbeat/module/kubernetes/controllermanager" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 21fb5c7958db..861ff13108f5 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -235,6 +235,7 @@ metricbeat.modules: #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true diff --git a/metricbeat/module/docker/_meta/config.reference.yml b/metricbeat/module/docker/_meta/config.reference.yml index f651b1385afa..f34e21c7fed8 100644 --- a/metricbeat/module/docker/_meta/config.reference.yml +++ b/metricbeat/module/docker/_meta/config.reference.yml @@ -9,6 +9,7 @@ #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml index 340f2106bfc5..ffe51d651fa4 100644 --- a/metricbeat/module/docker/_meta/config.yml +++ b/metricbeat/module/docker/_meta/config.yml @@ -8,6 +8,7 @@ # - info # - memory # - network + # - network_summary period: 10s hosts: ["unix:///var/run/docker.sock"] diff --git a/metricbeat/module/docker/fields.go b/metricbeat/module/docker/fields.go index 994f6bd7cb18..ef99e2d135d9 100644 --- a/metricbeat/module/docker/fields.go +++ b/metricbeat/module/docker/fields.go @@ -32,5 +32,5 @@ func init() { // AssetDocker returns asset data. // This is the base64 encoded gzipped contents of module/docker. func AssetDocker() string { - return "eJzsXEuP47gRvvevKHQOARbTNhIscvAhwKZngmkks9OYR3IIAi9NlW3GFKkhKXs8vz7gQ7IsUZIfsrt70T6aVvFjPb4qkiXfwQq3E0gkXaG6ATDMcJzA7Vv3xe0NQIKaKpYZJsUE/noDAOAHQRtiNFDJOVKDCcyVTMPY6AZAIUeicQILcgOg0RgmFnoC/7nVmt/+1363lMpMqRRztpjAnHCNNwBzhjzREzfTHQiSYgWf/ZhtZqUqmWfhmwhG+3kQc6lSYr8GIhIHmGnDqAYyk7kJYv+oQeVCMLEAKoUhTKDSoyCliqaKqPxlORID1gGuoshSFqRoFKPl5Pazr8biU4e1Dy1NiUj2xgpwK9xupKqPdUC0n3svEMySGNgQDfgdaW5NzgSYJTbWMYrjUkgMxnElxOBxoN4Sg7BZokewU6HFF2aKw7BekOshtVNM7SXHZ2XZlCSJQq0xPjfLTp324RFK0S1LZj/q2o376nHLZT8w5rHQ4p9VREpKM53XNbEDxqVYRAZ7sLnPF2kI9+DkHAjnzkHmjKMu/LXFUfcAbi6B7XNAtUPkYmpJ1ggzRFF4LkgFdEnEAhPQTFD0A0yKuIENWQzo0Q8pWaCTOWryXpafw3ifcmFYinD/+HUYsluhEshHGTXR9WtKOCbTOZek/gOfGyaQoaIo6qM9Knr0D1k9WXPaJTERwIDOCMW4oQJcIVX6DDGDxUU4+4EJzLbOS0WezlDZB6zJqFRtHBNWZhhdxV0xEjZ9VPP4FZy8w3Srt9rgk6vVsY9H7hVstRigdcF+Di7Rgf0c1wgrHNo1AjAnNj5xrlE9tU6DJi2ULud1UJ+DDzTwnmN5t6qLUUKfTp0/X12fX8ooyjVZdEJ7EnvX8J1jXjs0+ql1BXL2P2wM+S+n1/TpfUZj2uPuWlGnYZ71st4MYM/2gO1f+vEh/eseujK4I4YqTwOYXjF51sab6RU8jD8OU4MqJPFd7Qnbq18ozdOcu02AlashyRUTC2dIzubl9iF2ANEGtApWZpfYde2MeBLoHbzZ1jQ2yL0Ai6Bqe/iABfzNPurAn45dNQ8xeqEfpVuaK4XCBB1nNvshlY2jnkrlhWrNKE4tTVwAmU8ljoOMLCaDh4+g8FuO2ug3NpIFEdLjbNqmALohzAyAEvpgFsBAZ1aRdlpraybgW445autKxUIOBu8ebRphKP3u6NtPVC6ilYzi5N2RlRLMFFJLOhP4y+jnUwn8IP8sTa5YI1wGoU0n+MXx5mmonwtxWvQGxQvgzqDnV/J8Jc+IJp1zPDF7dnto6Z15mhK1vVzdScRLpVJb2MsMlTswf7GU6mrRwggvg1srSn/l11d+bYJxx15PRK8NVou4aIET1/tnM6fe7NflHH+2MPRt9TuLKCa1vKgesHHAT8YSf09P1oRxMuMYnXeuZDr4MmWuaHw6K3u46b4s0T1u/cwfhwGmzHW8OI6u+8EOB6FW5mWQdM4q6/nijNKhKayvBGh4Wd+yD8BRLP/hbZEaDzPFnmKMUWyWd+X96Bko1M9Bz1rFv4hiMtdWyHhNeI4VXPtre2PZEUViVycFMKP3PbtY1xIJN0u6RLoagNYq0ponqHsPvK/8MiGGwIZxDlLwLcxwxwi+dyyptRFpyxsK7XL3hIbf/fb+3S///PL+/v27+3/8Bkxoo3IXTbAk2rdT5BoTm/1nOeOJU1t4lqW1q5njmXlOGGdioY1CsorGEhMGF426rMf+VAqau2rKToAJ1I12udxQNZaXDVQmcf6MhdHJDOKEBWUf20mEIplGusegq7XsAEh1faBI4pIqxlDmGkjcRN1YZG6yPEZRA3BTFUrLPKVpvjMzbXhQFUk8Qk5TSsNdy1xjY30A1nNyhrkxaimyTggdm/A8ME62ljJZgsKwOWs2t/VFUtjFXcZt4Ktg3/IC6w4kLNjaEnUWsle8z60KMyMXRPmwAxbyrAP8BtgcmLEe7TaOLl1tlowu/RY87H/94hKmkBq+dROiqFPaBdthXZOu3UmWfbEeUX9P7IANor570PVfepc81g/XTJm8sU2EofsvGyXAPgqFi5yTGDUN3KFqsRDOgRK6xMTD0kC0lpS54zgjm07WUm4V4DmZIT/1Cv+MnlE/bw+46zWrMjE/q03gQcxlQfgwI7aYtNWlMZmejMeJpHrk68kRlekYxYIJHCuco0JBcUwyNvbjU4WpNDglGZuu/zT688/jP4wTpjNOtne+je1uwxK8Y7s3Fs59B6CooYcK649rVM5N99rdjw7ujNia/AJRVT+O8hNF3uhoYgpvf1wBVPt7Jk1U2sgsu4qqwkwHoYqd4F0Ck8u0Xaq6xHlVqFGKXa7UJlpOxXE43o5iOb4nqlUbfpYm06WYyr3LoKO57oOTMEx565nhpxPzT0RdlfFpSrKMiUX48e1Pt8ep9hPZBG2FF9hcLecSrNOWDqMjO+o2KGpOWg4RqUxTNtgu+N5JM65vzx30CPg3E4nc1L2qj2NPitEB7q2818YFlPzfPCy5BrRHJKtgrl4Fl1AVWxOD041UK+twGs2o/QIjgr0Ldw/mMDeEuUGj6cU7J4yPqMxbzmU6b1g6wfydMJv3cxsLcRLmrC0OhlVLICk3XRyJ0oNVPJ8+f95jimNLnacNw4BcoXYpzHqQ23J0bKyjp9q9zgO9Xc0H4v7QgriQ2tpM7/rYh7L6V+1PeU63e0q+P4HVP5DvBerIewfw/Ozs3z5osy08t0CqKbVRgQk0lqzPKcF+9SJOr8HihWm0hDmrTi6AlqLdVC31efxyMx6ZZ1z0PwgqU5sqgyFCdbe75D82jJ+q+6de+7NiYU5me5Qkqnu/2BPaJyALM+4QZoSusEmYlRsBpWTjSAIG2z568e4i9GBI4QdX2NJ2Y6rc3VwnYD7mZiF/jwEji4U924ApET6fgDkc0vUCphvTLsHMZN7ydyenXF/E84j/F4b9vxpxN7H1K5WXEydDJZbhDP6aUC6TUAYNkJa88TsMkKESyfAB8ppATkwg/w8AAP//JPFoyQ==" + return "eJzsXE1v4zjSvudXFPIeXqCR2NjFYA8+LDCb7kUHOz0ddCe7h8HAQ1Nlu9YSqSYpu92/fkFSkm2JkvwhO8kgPuQQWlVPfRfJkm9hgesRRJIvUF0BGDIxjuD6vfvH9RVAhJorSg1JMYK/XwEA+EXQhhkNXMYxcoMRTJVM8rXBFYDCGJnGEczYFYBGY0jM9Ah+u9Y6vv7d/m8ulRlzKaY0G8GUxRqvAKaEcaRHjtMtCJbgFj77MevUUlUyS/P/BDDaz72YSpUw+29gInKASRviGthEZiYn+/8aVCYEiRlwKQwjgUoPcirbaLYRld8sV0LAWsBtKbKkBQkaRbxkbj+7aiw+VVi70JKEiWhnrQC3wPVKqupaC0T7ufMEwcyZgRXTgN+RZ9bkJMDMsSbHIIxLITMYxhUxg4eBes8MwmqOHsFGhRZfzikMw3pBpvvUTsHaUw5zpXTMokih1hjmTemxbO8foCTdIDL9qGo37KuHiUs/MOSx0OCf24iUlGY8rWpiAyyWYhZY7MDmPo/SsNiDk1NgcewcZEox6sJfGxx1B+DqHNi+5qg2iFxMzdkSYYIoCs8FqYDPmZhhBJoER79AUoQNbNisR4++T9gMHc1BPe+l2SkZ70smDCUIdw9P/SS7BSqB8SDlJii/5izGaDyNJat+wdeGEaSoOIrqaoeKHvxDVk/WnFYkEjkY0CnjGDZUDldIlbxAzGBxsZh+YASTtfNSkSUTVPYBazIuVVOOySUzxBdhVwyETVeqeXgCR28/3eq1NvjsanXZxyP3CrZazKG1wX4JLtGC/RTXyCXs2zVyYI5smHGmUT23TnNNWihtzuugvgQfqOE9xfJOqrOlhC6dOn++uD4fyyjKNJu1QnsWe1fwnWJeuzR41yiBnPwXa0v+n+NL+vRuRiPtcbdJ1GqYFy3WTQ/2bA7YbtEPD+lfd9CVwR0wVHkaQHpB8qSNN+kF3A8/99ODKmThXe0R26ufOc+SLHabAEtXQ5QpEjNnyJim5fYhdADRBHQbrEzPsevaGPEo0Bt4k7WpbZA7ARZB1fTwHgL8wz7qwB+PXdUPMTqhH6RbnimFwuQ6Tm31Qy5rRz1bnReqJXEc2zRxBmS+lLgcZGTBDO4/g8JvGWqjb2wkCyakx1m3TQF0xcj0gBK6YBbAQKdWkZattTUJ+JZhhtq6UiHI3uDdo3Uj9KXfTfr2jEohGpNROHm3VKUIU4XcJp0R/G3w07EJfC//LE2uqBYuvaRNR/jV5c3jUL+UxGnRGxSvIHfmen5Lnm/JM6BJ5xzPnD3bPbT0zixJmFqfr+9k4rWmUtvYyxSVOzB/tSnV9aKFEV5Hbt1S+lt+fcuvdTDu2OuZ0mstqwVctMCJy92zmWNv9qt0Dj9b6Pu2+oNFFKJaXlT3ODjgmVHk7+nZklHMJjEG+U6VTHoXU2aKh9lZ2v2xe5yje9z6mT8OA0zITby4HF31gw0Oxi3N8yBp5Sqr9eKE1qFOrKsFqHlZl9h74CjEv39flMb9TLGjGGMUTbK2uh88A4XqOehJUvybKZKZtkSGSxZnuIVrV7Ybmx1RRFY6KYCM3vXsQq45stjM+Rz5ooe0tkWtfoK688DHrW9GzDBYURyDFPEaJrjJCH52LKqMEWmbNxRacXeI5t/74+OHn395/Hj38cPdv/4AEtqozEUTzJn24xSZxshW/0lGceTUlj9LSeVq5vDMPGUUk5hpo5AtgrFEwuCs1pd12J9LwTPXTVkGGEHVaOerDdvG8rSByyicP0NhdHQGccRyZR86SYQiGgemx6BttGwPSFV9oIjClLaMocwlkDhG7VhkZtIslKJ6yE3bUBr4lKb5TmZc86BtJOEIOU4pNXcta42N9R6ynqPTz41RQ5N1ROjYgueBxWxtUyZFKAxNqT7c1hVJ+S7uPG4DT4K+ZQXWDUiY0dIm6jSvXuE5t22YKTsjyvsNsLzOOsA3QFMgYz3abRxduVrNic/9Fjzf/3rhIlLITbx2DFFUU9oZx2HdkK7dSZZzsR5R90xsjwOifnrQzV96lzzUD5ekTFbbJkLf85e1FmAXhcJZFrNQaup5QtViYXEMnPE5Rh6WBqa15OSO44ysO1lDu1WAj9kE42Ov8E+YGfV8O8BdbliVxPSkMYF7MZVFwocJs82k7S6NSfVoOIwk1wPfTw64TIYoZiRwqHCKCgXHIUtp6NfHChNpcMxSGi//MvjrT8P/G0ak05itb/0Y2+2KIrylzRsLp74DUPTQfYX15yUq56Y74+4HB3fKbE9+hqiqHkd5RoE3OuqY8rc/LgCq+T2TOiptZJpeRFU5p71QhU7wzoHJVdo2VZ3jvCrvUYpdrtQm2E6Fcbi8HcRy+ExUozY8l3qmSzCRO5dBB+e6T45CP+2tzwzvjqw/AXVtrY8TlqYkZvmXr99dH6baL2yVayt/gc31cq7AOm3pfHVgV90GRU1ZwyEil0lCve2C7xw14+b23EGPgP+QiOSq6lVdOfaoGO3h3sp7bZhAmf/rhyWXgPaAbJGbq1PBJVRFS2ZwvJJqYR1Ooxk0X2AEsLfh7sCc84acN2g0nXinjOIBl1nDuUzrDUsrmH8ysnU/s7EQTsIxNcVBv2rJk5RjF0aidG8dz5evX3cyxaGtzvOGYY5coXYlzHqQ23K0bKyDp9qdzgOdU8174v7UgLig2jhM7+bY+7L6k/anPMfbPWHfn8Hqn9j3AnXgvQN4eXb2bx802RZeWiBVlFrrwAQam6xPacF+9SSO78HCjWmwhTmpTy6AlqQdq4b+PHy5GY7MEy767wWXiS2VuSHy7m5zyX9oGD/X9E+196dCMEezOUoi1b5f7AjtI5DlHDcIU8YXWE+YWzcCSsnakQT0tn305N1F6N6Q8i9cYEvbjmnr7uYyAfM5MzP5ZwwYWQj2YgOmRPhyAmZ/SJcLmHZMmwIzkVnDz50cc30RriP+Vxh2f2rE3cRWr1ReT5z0VVj6M/hbQTlPQek1QBrqxp8wQPoqJP0HyFsBObGAVHZt4/rbFOH4KPdhEzRsv11dM4vWCZH0kPPzrpr2UBzfNYxl8z65Pd51sMuiPtk9ve9mN47JYM88fyGD7YyJJ71a8e5Th6T2r/89jLThWuyIq6g5ul/rgofNXG2Z4W+AS6VQp9KPoBoJw1RJPvwtpej3ocDw+PEGZ68wNwCL0lQy8pOg+e8h7A1fOwn+FwAA//+qunky" } diff --git a/metricbeat/module/docker/network/_meta/data.json b/metricbeat/module/docker/network/_meta/data.json index fba2fae388c3..a671884a1033 100644 --- a/metricbeat/module/docker/network/_meta/data.json +++ b/metricbeat/module/docker/network/_meta/data.json @@ -1,36 +1,48 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "container": { - "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80", "image": { - "name": "metricbeat_elasticsearch" + "name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT" }, - "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "name": "elastic-package-stack_elastic-agent_1", "runtime": "docker" }, "docker": { "container": { "labels": { - "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", + "com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", - "com_docker_compose_project": "metricbeat", - "com_docker_compose_service": "elasticsearch", - "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", - "com_docker_compose_version": "1.23.1", + "com_docker_compose_project": "elastic-package-stack", + "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml", + "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport", + "com_docker_compose_service": "elastic-agent", + "com_docker_compose_version": "1.27.4", + "description": "Agent manages other beats based on configuration provided.", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "io_k8s_display-name": "Elastic-Agent image", "license": "Elastic License", - "org_label-schema_build-date": "20181006", - "org_label-schema_license": "GPLv2", - "org_label-schema_name": "elasticsearch", + "maintainer": "infra@elastic.co", + "name": "elastic-agent", + "org_label-schema_build-date": "2021-04-21T06:45:43Z", + "org_label-schema_license": "Elastic License", + "org_label-schema_name": "elastic-agent", "org_label-schema_schema-version": "1.0", - "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", - "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf", + "org_label-schema_vcs-url": "github.com/elastic/beats/v7", "org_label-schema_vendor": "Elastic", - "org_label-schema_version": "6.5.1" + "org_label-schema_version": "7.13.0-SNAPSHOT", + "org_opencontainers_image_created": "2021-04-21T06:45:43Z", + "org_opencontainers_image_licenses": "Elastic License", + "org_opencontainers_image_title": "Elastic-Agent", + "org_opencontainers_image_vendor": "Elastic", + "release": "1", + "summary": "elastic-agent", + "url": "https://www.elastic.co/beats/elastic-agent", + "vendor": "Elastic", + "version": "7.13.0-SNAPSHOT" } }, "network": { @@ -41,10 +53,10 @@ "packets": 0 }, "inbound": { - "bytes": 23047, + "bytes": 156469392, "dropped": 0, "errors": 0, - "packets": 241 + "packets": 710866 }, "interface": "eth0", "out": { @@ -54,10 +66,10 @@ "packets": 0 }, "outbound": { - "bytes": 0, + "bytes": 3150916813, "dropped": 0, "errors": 0, - "packets": 0 + "packets": 1059447 } } }, @@ -67,7 +79,8 @@ "module": "docker" }, "metricset": { - "name": "network" + "name": "network", + "period": 10000 }, "service": { "address": "/var/run/docker.sock", diff --git a/metricbeat/module/docker/network_summary/_meta/data.json b/metricbeat/module/docker/network_summary/_meta/data.json new file mode 100644 index 000000000000..769fa82dd3a8 --- /dev/null +++ b/metricbeat/module/docker/network_summary/_meta/data.json @@ -0,0 +1,296 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "container": { + "id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80", + "image": { + "name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT" + }, + "name": "elastic-package-stack_elastic-agent_1", + "runtime": "docker" + }, + "docker": { + "container": { + "labels": { + "com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3", + "com_docker_compose_container-number": "1", + "com_docker_compose_oneoff": "False", + "com_docker_compose_project": "elastic-package-stack", + "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml", + "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport", + "com_docker_compose_service": "elastic-agent", + "com_docker_compose_version": "1.27.4", + "description": "Agent manages other beats based on configuration provided.", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "io_k8s_display-name": "Elastic-Agent image", + "license": "Elastic License", + "maintainer": "infra@elastic.co", + "name": "elastic-agent", + "org_label-schema_build-date": "2021-04-21T06:45:43Z", + "org_label-schema_license": "Elastic License", + "org_label-schema_name": "elastic-agent", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf", + "org_label-schema_vcs-url": "github.com/elastic/beats/v7", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "7.13.0-SNAPSHOT", + "org_opencontainers_image_created": "2021-04-21T06:45:43Z", + "org_opencontainers_image_licenses": "Elastic License", + "org_opencontainers_image_title": "Elastic-Agent", + "org_opencontainers_image_vendor": "Elastic", + "release": "1", + "summary": "elastic-agent", + "url": "https://www.elastic.co/beats/elastic-agent", + "vendor": "Elastic", + "version": "7.13.0-SNAPSHOT" + } + }, + "network_summary": { + "icmp": { + "InAddrMaskReps": 0, + "InAddrMasks": 0, + "InCsumErrors": 0, + "InDestUnreachs": 0, + "InEchoReps": 0, + "InEchos": 0, + "InErrors": 0, + "InMsgs": 0, + "InParmProbs": 0, + "InRedirects": 0, + "InSrcQuenchs": 0, + "InTimeExcds": 0, + "InTimestampReps": 0, + "InTimestamps": 0, + "OutAddrMaskReps": 0, + "OutAddrMasks": 0, + "OutDestUnreachs": 0, + "OutEchoReps": 0, + "OutEchos": 0, + "OutErrors": 0, + "OutMsgs": 0, + "OutParmProbs": 0, + "OutRedirects": 0, + "OutSrcQuenchs": 0, + "OutTimeExcds": 0, + "OutTimestampReps": 0, + "OutTimestamps": 0 + }, + "ip": { + "DefaultTTL": 64, + "ForwDatagrams": 0, + "Forwarding": 1, + "FragCreates": 0, + "FragFails": 0, + "FragOKs": 0, + "InAddrErrors": 0, + "InBcastOctets": 0, + "InBcastPkts": 0, + "InCEPkts": 0, + "InCsumErrors": 0, + "InDelivers": 1246323, + "InDiscards": 0, + "InECT0Pkts": 0, + "InECT1Pkts": 0, + "InHdrErrors": 0, + "InMcastOctets": 0, + "InMcastPkts": 0, + "InNoECTPkts": 1246337, + "InNoRoutes": 0, + "InOctets": 172967508, + "InReceives": 1246323, + "InTruncatedPkts": 0, + "InUnknownProtos": 0, + "OutBcastOctets": 0, + "OutBcastPkts": 0, + "OutDiscards": 0, + "OutMcastOctets": 0, + "OutMcastPkts": 0, + "OutNoRoutes": 0, + "OutOctets": 2783525210, + "OutRequests": 1550542, + "ReasmFails": 0, + "ReasmOKs": 0, + "ReasmOverlaps": 0, + "ReasmReqds": 0, + "ReasmTimeout": 0 + }, + "namespace": { + "ID": 4026532414, + "pid": 2240728 + }, + "tcp": { + "ActiveOpens": 742, + "ArpFilter": 0, + "AttemptFails": 0, + "BusyPollRxPackets": 0, + "CurrEstab": 12, + "DelayedACKLocked": 0, + "DelayedACKLost": 1, + "DelayedACKs": 23466, + "EmbryonicRsts": 0, + "EstabResets": 0, + "IPReversePathFilter": 0, + "InCsumErrors": 0, + "InErrs": 0, + "InSegs": 1243467, + "ListenDrops": 0, + "ListenOverflows": 0, + "LockDroppedIcmps": 0, + "MaxConn": -1, + "OfoPruned": 0, + "OutOfWindowIcmps": 0, + "OutRsts": 709, + "OutSegs": 2936174, + "PAWSActive": 0, + "PAWSEstab": 19, + "PFMemallocDrop": 0, + "PassiveOpens": 4, + "PruneCalled": 0, + "RcvPruned": 0, + "RetransSegs": 1613, + "RtoAlgorithm": 1, + "RtoMax": 120000, + "RtoMin": 200, + "SyncookiesFailed": 0, + "SyncookiesRecv": 0, + "SyncookiesSent": 0, + "TCPACKSkippedChallenge": 0, + "TCPACKSkippedFinWait2": 0, + "TCPACKSkippedPAWS": 0, + "TCPACKSkippedSeq": 0, + "TCPACKSkippedSynRecv": 0, + "TCPACKSkippedTimeWait": 0, + "TCPAbortFailed": 0, + "TCPAbortOnClose": 0, + "TCPAbortOnData": 705, + "TCPAbortOnLinger": 0, + "TCPAbortOnMemory": 0, + "TCPAbortOnTimeout": 0, + "TCPAckCompressed": 0, + "TCPAutoCorking": 763, + "TCPBacklogCoalesce": 1358, + "TCPBacklogDrop": 0, + "TCPChallengeACK": 0, + "TCPDSACKIgnoredDubious": 0, + "TCPDSACKIgnoredNoUndo": 1498, + "TCPDSACKIgnoredOld": 0, + "TCPDSACKOfoRecv": 0, + "TCPDSACKOfoSent": 0, + "TCPDSACKOldSent": 1, + "TCPDSACKRecv": 1586, + "TCPDSACKRecvSegs": 1589, + "TCPDSACKUndo": 2, + "TCPDeferAcceptDrop": 0, + "TCPDelivered": 2275375, + "TCPDeliveredCE": 0, + "TCPFastOpenActive": 0, + "TCPFastOpenActiveFail": 0, + "TCPFastOpenBlackhole": 0, + "TCPFastOpenCookieReqd": 0, + "TCPFastOpenListenOverflow": 0, + "TCPFastOpenPassive": 0, + "TCPFastOpenPassiveAltKey": 0, + "TCPFastOpenPassiveFail": 0, + "TCPFastRetrans": 7, + "TCPFromZeroWindowAdv": 0, + "TCPFullUndo": 1, + "TCPHPAcks": 537452, + "TCPHPHits": 144087, + "TCPHystartDelayCwnd": 0, + "TCPHystartDelayDetect": 0, + "TCPHystartTrainCwnd": 36, + "TCPHystartTrainDetect": 2, + "TCPKeepAlive": 171573, + "TCPLossFailures": 0, + "TCPLossProbeRecovery": 0, + "TCPLossProbes": 2183, + "TCPLossUndo": 0, + "TCPLostRetransmit": 0, + "TCPMD5Failure": 0, + "TCPMD5NotFound": 0, + "TCPMD5Unexpected": 0, + "TCPMTUPFail": 0, + "TCPMTUPSuccess": 0, + "TCPMemoryPressures": 0, + "TCPMemoryPressuresChrono": 0, + "TCPMinTTLDrop": 0, + "TCPOFODrop": 0, + "TCPOFOMerge": 0, + "TCPOFOQueue": 0, + "TCPOrigDataSent": 2273068, + "TCPPartialUndo": 1, + "TCPPureAcks": 215589, + "TCPRcvCoalesce": 10, + "TCPRcvCollapsed": 0, + "TCPRcvQDrop": 0, + "TCPRenoFailures": 0, + "TCPRenoRecovery": 0, + "TCPRenoRecoveryFail": 0, + "TCPRenoReorder": 0, + "TCPReqQFullDoCookies": 0, + "TCPReqQFullDrop": 0, + "TCPRetransFail": 0, + "TCPSACKDiscard": 0, + "TCPSACKReneging": 0, + "TCPSACKReorder": 256, + "TCPSYNChallenge": 0, + "TCPSackFailures": 0, + "TCPSackMerged": 5, + "TCPSackRecovery": 4, + "TCPSackRecoveryFail": 0, + "TCPSackShiftFallback": 160, + "TCPSackShifted": 0, + "TCPSlowStartRetrans": 0, + "TCPSpuriousRTOs": 0, + "TCPSpuriousRtxHostQueues": 2, + "TCPSynRetrans": 24, + "TCPTSReorder": 1, + "TCPTimeWaitOverflow": 0, + "TCPTimeouts": 24, + "TCPToZeroWindowAdv": 0, + "TCPWantZeroWindowAdv": 0, + "TCPWinProbe": 0, + "TCPWqueueTooBig": 0, + "TCPZeroWindowDrop": 0, + "TW": 3, + "TWKilled": 0, + "TWRecycled": 0, + "TcpDuplicateDataRehash": 0, + "TcpTimeoutRehash": 24 + }, + "udp": { + "IgnoredMulti": 0, + "InCsumErrors": 0, + "InDatagrams": 2856, + "InErrors": 0, + "NoPorts": 0, + "OutDatagrams": 2856, + "RcvbufErrors": 0, + "SndbufErrors": 0 + }, + "udp_lite": { + "IgnoredMulti": 0, + "InCsumErrors": 0, + "InDatagrams": 0, + "InErrors": 0, + "NoPorts": 0, + "OutDatagrams": 0, + "RcvbufErrors": 0, + "SndbufErrors": 0 + } + } + }, + "event": { + "dataset": "docker.network_summary", + "duration": 115000, + "module": "docker" + }, + "metricset": { + "name": "network_summary", + "period": 10000 + }, + "service": { + "address": "unix:///var/run/docker.sock", + "type": "docker" + } +} \ No newline at end of file diff --git a/metricbeat/module/docker/network_summary/_meta/docs.asciidoc b/metricbeat/module/docker/network_summary/_meta/docs.asciidoc new file mode 100644 index 000000000000..3d3597e8e127 --- /dev/null +++ b/metricbeat/module/docker/network_summary/_meta/docs.asciidoc @@ -0,0 +1,4 @@ +The `network_summary` metricset collects detailed network metrics from the processes associated with a container. +These stats come from `/proc/[PID]/net`, and are summed across the different namespaces found across the PIDs. + +Because this metricset will try to access network counters from procfs, it is only available on linux. \ No newline at end of file diff --git a/metricbeat/module/docker/network_summary/_meta/fields.yml b/metricbeat/module/docker/network_summary/_meta/fields.yml new file mode 100644 index 000000000000..7533a77e5b4c --- /dev/null +++ b/metricbeat/module/docker/network_summary/_meta/fields.yml @@ -0,0 +1,34 @@ +- name: network_summary + type: group + release: beta + description: > + network_summary + fields: + - name: ip.* + type: object + description: > + IP counters + - name: tcp.* + type: object + description: > + TCP counters + - name: udp.* + type: object + description: > + UDP counters + - name: udp_lite.* + type: object + description: > + UDP Lite counters + - name: icmp.* + type: object + description: > + ICMP counters + - name: namespace.pid + type: long + description: > + The root PID of the container, corresponding to /proc/[pid]/net + - name: namespace.id + type: long + description: > + The ID of the network namespace used by the container, corresponding to /proc/[pid]/ns/net \ No newline at end of file diff --git a/metricbeat/module/docker/network_summary/config.go b/metricbeat/module/docker/network_summary/config.go new file mode 100644 index 000000000000..62e355113b6a --- /dev/null +++ b/metricbeat/module/docker/network_summary/config.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package network_summary + +import "github.com/elastic/beats/v7/metricbeat/module/docker" + +//Config is the config object for docker/network +type Config struct { + TLS *docker.TLSConfig `config:"ssl"` + DeDot bool `config:"labels.dedot"` + NetworkSummary bool `config:"network.network_summary"` +} + +// DefaultConfig returns default module config +func DefaultConfig() Config { + return Config{ + DeDot: true, + NetworkSummary: false, + } +} diff --git a/metricbeat/module/docker/network_summary/doc.go b/metricbeat/module/docker/network_summary/doc.go new file mode 100644 index 000000000000..917cff9bf6c7 --- /dev/null +++ b/metricbeat/module/docker/network_summary/doc.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package network_summary diff --git a/metricbeat/module/docker/network_summary/helper.go b/metricbeat/module/docker/network_summary/helper.go new file mode 100644 index 000000000000..adb0fdcc75cb --- /dev/null +++ b/metricbeat/module/docker/network_summary/helper.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package network_summary + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "time" + + "github.com/docker/docker/client" + "github.com/pkg/errors" + + "github.com/elastic/go-sysinfo" + + sysinfotypes "github.com/elastic/go-sysinfo/types" +) + +// NoSumStats lists "stats", often config/state values, that can't be safely summed across PIDs +var NoSumStats = []string{ + "RtoAlgorithm", + "RtoMin", + "RtoMax", + "MaxConn", + "Forwarding", + "DefaultTTL", +} + +var nsRegex = regexp.MustCompile(`\d+`) + +// fetchContainerNetStats gathers the PIDs associated with a container, and then uses go-sysinfo to grab the /proc/[pid]/net counters and sum them across PIDs. +func fetchContainerNetStats(client *client.Client, timeout time.Duration, container string) (*sysinfotypes.NetworkCountersInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + inspect, err := client.ContainerInspect(ctx, container) + if err != nil { + return nil, errors.Wrapf(err, "error fetching stats for container %s", container) + } + rootPID := inspect.ContainerJSONBase.State.Pid + + proc, err := sysinfo.Process(rootPID) + procNet, ok := proc.(sysinfotypes.NetworkCounters) + if !ok { + return nil, errors.Wrapf(err, "cannot fetch network counters for PID %d", rootPID) + } + + counters, err := procNet.NetworkCounters() + if err != nil { + return &sysinfotypes.NetworkCountersInfo{}, errors.Wrapf(err, "error fetching network counters for PID %d", rootPID) + } + + return counters, nil + +} + +// fetch the network namespace associated with the PID. +func fetchNamespace(pid int) (int, error) { + nsLink, err := os.Readlink(filepath.Join("/proc/", fmt.Sprintf("%d", pid), "/ns/net")) + if err != nil { + return 0, errors.Wrap(err, "error reading network namespace link") + } + nsidString := nsRegex.FindString(nsLink) + // This is minor metadata, so don't consider it an error + if nsidString == "" { + return 0, nil + } + + nsID, err := strconv.Atoi(nsidString) + if err != nil { + return 0, errors.Wrapf(err, "error converting %s to int", nsidString) + } + return nsID, nil +} diff --git a/metricbeat/module/docker/network_summary/network_summary.go b/metricbeat/module/docker/network_summary/network_summary.go new file mode 100644 index 000000000000..8ed3d4c87ef5 --- /dev/null +++ b/metricbeat/module/docker/network_summary/network_summary.go @@ -0,0 +1,129 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package network_summary + +import ( + "context" + "runtime" + + "github.com/docker/docker/client" + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/metric/system/network" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/docker" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("docker", "network_summary", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + dockerClient *client.Client + cfg Config +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The docker network_summary metricset is beta.") + + config := DefaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + client, err := docker.NewDockerClient(base.HostData().URI, docker.Config{DeDot: config.DeDot, TLS: config.TLS}) + if err != nil { + return nil, err + } + + // Network summary requres a linux procfs system under it to read from the cgroups. Disable reporting otherwise. + if runtime.GOOS != "linux" { + base.Logger().Debug("Not running on linux, docker network detailed stats disabled.") + config.NetworkSummary = false + } + + return &MetricSet{ + BaseMetricSet: base, + dockerClient: client, + cfg: config, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error { + + stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) + if err != nil { + return errors.Wrap(err, "failed to get docker stats") + } + + for _, myStats := range stats { + + ctx, cancel := context.WithTimeout(ctx, m.Module().Config().Timeout) + defer cancel() + + inspect, err := m.dockerClient.ContainerInspect(ctx, myStats.Container.ID) + if err != nil { + return errors.Wrapf(err, "error fetching stats for container %s", myStats.Container.ID) + } + + rootPID := inspect.ContainerJSONBase.State.Pid + + netNS, err := fetchNamespace(rootPID) + if err != nil { + return errors.Wrapf(err, "error fetching namespace for PID %d", rootPID) + } + + networkStats, err := fetchContainerNetStats(m.dockerClient, m.Module().Config().Timeout, myStats.Container.ID) + if err != nil { + return errors.Wrap(err, "error fetching per-PID stats") + } + + summary := network.MapProcNetCounters(networkStats) + // attach metadata associated with the network counters + summary["namespace"] = common.MapStr{ + "id": netNS, + "pid": rootPID, + } + + report.Event(mb.Event{ + RootFields: docker.NewContainer(myStats.Container, m.cfg.DeDot).ToMapStr(), + MetricSetFields: summary, + }) + + } + + return nil +} diff --git a/metricbeat/module/docker/network_summary/network_summary_test.go b/metricbeat/module/docker/network_summary/network_summary_test.go new file mode 100644 index 000000000000..fdf6d100f727 --- /dev/null +++ b/metricbeat/module/docker/network_summary/network_summary_test.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration linux + +package network_summary + +import ( + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + ms := mbtest.NewFetcher(t, getConfig()) + ms.WriteEvents(t, "") +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "docker", + "metricsets": []string{"network_summary"}, + "hosts": []string{"unix:///var/run/docker.sock"}, + } +} diff --git a/metricbeat/module/system/network_summary/network_summary.go b/metricbeat/module/system/network_summary/network_summary.go index bbb27d589b04..ff08d6731e0c 100644 --- a/metricbeat/module/system/network_summary/network_summary.go +++ b/metricbeat/module/system/network_summary/network_summary.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/metric/system/network" "github.com/elastic/beats/v7/metricbeat/mb" sysinfo "github.com/elastic/go-sysinfo" sysinfotypes "github.com/elastic/go-sysinfo/types" @@ -70,7 +71,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } report.Event(mb.Event{ - MetricSetFields: eventMapping(counterInfo), + MetricSetFields: network.MapProcNetCounters(counterInfo), }) return nil diff --git a/metricbeat/module/system/network_summary/network_summary_test.go b/metricbeat/module/system/network_summary/network_summary_test.go index bd2ba95d0068..bac14f519217 100644 --- a/metricbeat/module/system/network_summary/network_summary_test.go +++ b/metricbeat/module/system/network_summary/network_summary_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/network" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/go-sysinfo/types" ) @@ -50,7 +51,7 @@ func TestMapping(t *testing.T) { "udp": map[string]uint64{"IgnoredMulti": 10}, "udp_lite": map[string]uint64{"IgnoredMulti": 0}} - out := eventMapping(example) + out := network.MapProcNetCounters(example) assert.Equal(t, exampleOut, out) } diff --git a/metricbeat/modules.d/docker.yml.disabled b/metricbeat/modules.d/docker.yml.disabled index f414279857e6..cd2b5a8aaa63 100644 --- a/metricbeat/modules.d/docker.yml.disabled +++ b/metricbeat/modules.d/docker.yml.disabled @@ -11,6 +11,7 @@ # - info # - memory # - network + # - network_summary period: 10s hosts: ["unix:///var/run/docker.sock"] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index eb8007437435..6fedb3b78d9c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -460,6 +460,7 @@ metricbeat.modules: #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true