libbeat: override /sys/fs/cgroup paths in Docker

When running in Docker, make sure metrics are read from `/sys/fs/cgroup/<subsystem>`, ignoring any paths in `/proc/self/cgroup`. Fixes #22844
elastic · Dec 4, 2020 · 2ef1aa8 · 2ef1aa8
1 parent de28183
commit 2ef1aa8
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 4 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -218,6 +218,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix FileVersion contained in Windows exe files. {pull}22581[22581]
 - Fix index template loading when the new index format is selected. {issue}22482[22482] {pull}22682[22682]
 - Use PROGRAMDATA environment variable instead of C:\ProgramData for windows install service {pull}22874[22874]
+- Fix reporting of cgroup metrics when running under Docker {pull}22879[22879]
 
 
 *Auditbeat*

diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl
@@ -79,6 +79,11 @@ USER {{ .user }}
 EXPOSE {{ $port }}
 {{- end }}
 
+# When running under Docker, we must ensure libbeat monitoring pulls cgroup
+# metrics from /sys/fs/cgroup/<subsystem>/, ignoring any paths found in
+# /proc/self/cgroup.
+ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/
+
 WORKDIR {{ $beatHome }}
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
 CMD ["-environment", "container"]
diff --git a/go.mod b/go.mod
@@ -72,7 +72,7 @@ require (
 	github.com/elastic/go-txfile v0.0.7
 	github.com/elastic/go-ucfg v0.8.3
 	github.com/elastic/go-windows v1.0.1 // indirect
-	github.com/elastic/gosigar v0.12.0
+	github.com/elastic/gosigar v0.13.0
 	github.com/fatih/color v1.5.0
 	github.com/fsnotify/fsevents v0.1.1
 	github.com/fsnotify/fsnotify v1.4.7

diff --git a/go.sum b/go.sum
@@ -278,8 +278,8 @@ github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+F
 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
-github.com/elastic/gosigar v0.12.0 h1:AsdhYCJlTudhfOYQyFNgx+fIVTfrDO0V1ST0vHgiapU=
-github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/gosigar v0.13.0 h1:EIeuQcLPKia759s6mlVztlxUyKiKYHo6y6kOODOLO7A=
+github.com/elastic/gosigar v0.13.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec h1:rAHd7DeHIHjSzvnkl197GKh9TCWGKg/z2BBbbGOEiWI=
 github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=

diff --git a/libbeat/cmd/instance/metrics.go b/libbeat/cmd/instance/metrics.go
@@ -21,6 +21,7 @@ package instance
 
 import (
 	"fmt"
+	"os"
 	"runtime"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -36,6 +37,11 @@ var (
 	systemMetrics    *monitoring.Registry
 )
 
+// libbeatMonitoringCgroupsHierarchyOverride is an undocumented environment variable which
+// overrides the cgroups path under /sys/fs/cgroup, which should be set to "/" when running
+// Beats under Docker.
+const libbeatMonitoringCgroupsHierarchyOverride = "LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE"
+
 func init() {
 	systemMetrics = monitoring.Default.NewRegistry("system")
 }
@@ -277,7 +283,10 @@ func reportBeatCgroups(_ monitoring.Mode, V monitoring.Visitor) {
 		return
 	}
 
-	cgroups, err := cgroup.NewReader("", true)
+	cgroups, err := cgroup.NewReaderOptions(cgroup.ReaderOptions{
+		IgnoreRootCgroups:        true,
+		CgroupsHierarchyOverride: os.Getenv(libbeatMonitoringCgroupsHierarchyOverride),
+	})
 	if err != nil {
 		if err == cgroup.ErrCgroupsMissing {
 			logp.Warn("cgroup data collection disabled: %v", err)