Fix behavior of hostfs under agent (elastic#28546)

* first pass at fixing hostfs config issues on agent

* add log line

* wrap global var

* fix tab in config example

* remove some dangling hostfs references

* clean up rest of linux

* fix hostfs check logic

libbeat/metric/system/cgroup/util.go

@@ -29,7 +29,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
-	"github.com/elastic/beats/v7/libbeat/paths"
 )
 
 var (
@@ -273,14 +272,14 @@ func (r Reader) ProcessCgroupPaths(pid int) (PathList, error) {
 			// If it's not set, warn the user that they've hit this.
 			controllerPath := filepath.Join(r.cgroupMountpoints.V2Loc, path)
 			// Depending on the test environment, Hostfs can either be blank, or `/`
-			if r.cgroupMountpoints.V2Loc == "" && len(paths.Paths.Hostfs) <= 1 {
+			if r.cgroupMountpoints.V2Loc == "" && len(r.rootfsMountpoint) <= 1 {
 				logp.L().Debugf(`PID %d contains a cgroups V2 path (%s) but no V2 mountpoint was found.
 This may be because metricbeat is running inside a container on a hybrid system.
 To monitor cgroups V2 processess in this way, mount the unified (V2) hierarchy inside
 the container as /sys/fs/cgroup/unified and start metricbeat with --system.hostfs.`, pid, line)
 				continue
-			} else if r.cgroupMountpoints.V2Loc == "" && len(paths.Paths.Hostfs) > 1 {
-				controllerPath = filepath.Join(paths.Paths.Hostfs, "/sys/fs/cgroup/unified", path)
+			} else if r.cgroupMountpoints.V2Loc == "" && len(r.rootfsMountpoint) > 1 {
+				controllerPath = filepath.Join(r.rootfsMountpoint, "/sys/fs/cgroup/unified", path)
 			}
 
 			cgpaths, err := ioutil.ReadDir(controllerPath)

libbeat/paths/paths.go

@@ -189,6 +189,6 @@ func Resolve(fileType FileType, path string) string {
 
 // String returns a textual representation
 func (paths *Path) String() string {
-	return fmt.Sprintf("Home path: [%s] Config path: [%s] Data path: [%s] Logs path: [%s]",
-		paths.Home, paths.Config, paths.Data, paths.Logs)
+	return fmt.Sprintf("Home path: [%s] Config path: [%s] Data path: [%s] Logs path: [%s] Hostfs Path: [%s]",
+		paths.Home, paths.Config, paths.Data, paths.Logs, paths.Hostfs)
 }

metricbeat/module/linux/conntrack/conntrack.go

@@ -50,12 +50,11 @@ type MetricSet struct {
 // any MetricSet specific configuration options if there are any.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	cfgwarn.Beta("The linux conntrack metricset is beta.")
-	linuxModule, ok := base.Module().(*linux.Module)
-	if !ok {
-		return nil, errors.New("unexpected module type")
-	}
 
-	path := filepath.Join(linuxModule.HostFS, "proc")
+	sys := base.Module().(linux.LinuxModule)
+	hostfs := sys.GetHostFS()
+
+	path := filepath.Join(hostfs, "proc")
 	newFS, err := procfs.NewFS(path)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error creating new Host FS at %s", path)

metricbeat/module/linux/ksm/ksm.go

@@ -50,14 +50,13 @@ type MetricSet struct {
 // any MetricSet specific configuration options if there are any.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	cfgwarn.Beta("The linux pageinfo metricset is beta.")
-	linuxModule, ok := base.Module().(*linux.Module)
-	if !ok {
-		return nil, errors.New("unexpected module type")
-	}
+
+	sys := base.Module().(linux.LinuxModule)
+	hostfs := sys.GetHostFS()
 
 	return &MetricSet{
 		BaseMetricSet: base,
-		fs:            filepath.Join(linuxModule.HostFS, "/sys/kernel/mm/ksm"),
+		fs:            filepath.Join(hostfs, "/sys/kernel/mm/ksm"),
 	}, nil
 }
 

metricbeat/module/linux/linux.go

@@ -20,6 +20,7 @@ package linux
 import (
 	"time"
 
+	"github.com/elastic/beats/v7/libbeat/paths"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
@@ -30,6 +31,10 @@ func init() {
 	}
 }
 
+type LinuxModule interface {
+	GetHostFS() string
+}
+
 // Module defines the base module config used in `linux`
 type Module struct {
 	mb.BaseModule
@@ -55,5 +60,14 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
 		dir = "/"
 	}
 
+	// Steer towards system.hostfs, since the two behave fundamentally the same, and system.hostfs has a CLI flag that many users may default to.
+	if len(paths.Paths.Hostfs) > 2 {
+		dir = paths.Paths.Hostfs
+	}
+
 	return &Module{BaseModule: base, HostFS: dir, Period: config.Period}, nil
 }
+
+func (m Module) GetHostFS() string {
+	return m.HostFS
+}

metricbeat/module/linux/pageinfo/pageinfo.go

@@ -51,14 +51,13 @@ type MetricSet struct {
 // any MetricSet specific configuration options if there are any.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	cfgwarn.Beta("The linux pageinfo metricset is beta.")
-	linuxModule, ok := base.Module().(*linux.Module)
-	if !ok {
-		return nil, errors.New("unexpected module type")
-	}
+
+	sys := base.Module().(linux.LinuxModule)
+	hostfs := sys.GetHostFS()
 
 	return &MetricSet{
 		BaseMetricSet: base,
-		fs:            linuxModule.HostFS,
+		fs:            hostfs,
 	}, nil
 }
 

metricbeat/module/linux/pressure/pressure.go

@@ -63,20 +63,18 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		return nil, fmt.Errorf("the %v/%v metricset is only supported on Linux", moduleName, metricsetName)
 	}
 
-	linuxModule, ok := base.Module().(*linux.Module)
-	if !ok {
-		return nil, errors.New("unexpected module type")
-	}
+	sys := base.Module().(linux.LinuxModule)
+	hostfs := sys.GetHostFS()
 
-	path := filepath.Join(linuxModule.HostFS, "proc")
+	path := filepath.Join(hostfs, "proc")
 	procfs, err := procfs.NewFS(path)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error creating new Host FS at %s", path)
 	}
 
 	return &MetricSet{
 		BaseMetricSet: base,
-		fs:            linuxModule.HostFS,
+		fs:            hostfs,
 		procfs:        procfs,
 	}, nil
 }

metricbeat/module/linux/rapl/rapl.go

@@ -37,8 +37,8 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/v7/libbeat/logp"
-	"github.com/elastic/beats/v7/libbeat/paths"
 	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/module/linux"
 )
 
 // init registers the MetricSet with the central registry as soon as the program
@@ -83,14 +83,16 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		return nil, err
 	}
 
-	CPUList, err := getMSRCPUs()
+	sys := base.Module().(linux.LinuxModule)
+	hostfs := sys.GetHostFS()
+	CPUList, err := getMSRCPUs(hostfs)
 	if err != nil {
 		return nil, errors.Wrap(err, "error getting list of CPUs to query")
 	}
 
 	// check to see if msr-safe is installed
 	if config.UseMSRSafe {
-		queryPath := filepath.Join(paths.Paths.Hostfs, "/dev/cpu/", fmt.Sprint(CPUList[0]), "msr_safe")
+		queryPath := filepath.Join(hostfs, "/dev/cpu/", fmt.Sprint(CPUList[0]), "msr_safe")
 		_, err := os.Stat(queryPath)
 		if errors.Is(err, os.ErrNotExist) {
 			return nil, errors.New("no msr_safe device found. Is the kernel module loaded?")
@@ -110,7 +112,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	handlers := map[int]rapl.RAPLHandler{}
 	for _, cpu := range CPUList {
-		formatPath := filepath.Join(paths.Paths.Hostfs, "/dev/cpu/%d")
+		formatPath := filepath.Join(hostfs, "/dev/cpu/%d")
 		if config.UseMSRSafe {
 			formatPath = filepath.Join(formatPath, "/msr_safe")
 		} else {
@@ -203,8 +205,8 @@ func (m *MetricSet) updatePower() map[int]map[rapl.RAPLDomain]energyUsage {
 
 // getMSRCPUs forms a list of CPU cores to query
 // For multi-processor systems, this will be more than 1.
-func getMSRCPUs() ([]int, error) {
-	CPUs, err := topoPkgCPUMap()
+func getMSRCPUs(hostfs string) ([]int, error) {
+	CPUs, err := topoPkgCPUMap(hostfs)
 	if err != nil {
 		return nil, errors.Wrap(err, "error fetching CPU topology")
 	}
@@ -225,12 +227,12 @@ func getMSRCPUs() ([]int, error) {
 //it is, however, the simplest way to do this. The intel power gadget iterates through each CPU using affinity masks, and runs `cpuid` in a loop to
 //figure things out
 //This uses /sys/devices/system/cpu/cpu*/topology/physical_package_id, which is what lscpu does. I *think* geopm does something similar to this.
-func topoPkgCPUMap() (map[int][]int, error) {
+func topoPkgCPUMap(hostfs string) (map[int][]int, error) {
 
 	sysdir := "/sys/devices/system/cpu/"
 	cpuMap := make(map[int][]int)
 
-	files, err := ioutil.ReadDir(filepath.Join(paths.Paths.Hostfs, sysdir))
+	files, err := ioutil.ReadDir(filepath.Join(hostfs, sysdir))
 	if err != nil {
 		return nil, err
 	}
@@ -240,7 +242,7 @@ func topoPkgCPUMap() (map[int][]int, error) {
 	for _, file := range files {
 		if file.IsDir() && re.MatchString(file.Name()) {
 
-			fullPkg := filepath.Join(paths.Paths.Hostfs, sysdir, file.Name(), "/topology/physical_package_id")
+			fullPkg := filepath.Join(hostfs, sysdir, file.Name(), "/topology/physical_package_id")
 			dat, err := ioutil.ReadFile(fullPkg)
 			if err != nil {
 				return nil, errors.Wrapf(err, "error reading file %s", fullPkg)

metricbeat/module/linux/rapl/rapl_test.go

@@ -24,8 +24,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-
-	"github.com/elastic/beats/v7/libbeat/paths"
 )
 
 // func TestData(t *testing.T) {
@@ -49,9 +47,9 @@ import (
 // }
 
 func TestTopo(t *testing.T) {
-	paths.Paths.Hostfs = "./testdata/"
+	hostfs := "./testdata/"
 
-	cpus, err := topoPkgCPUMap()
+	cpus, err := topoPkgCPUMap(hostfs)
 	assert.NoError(t, err)
 	good := map[int][]int{
 		0: {0, 1},

metricbeat/module/system/_meta/config.yml

@@ -17,8 +17,8 @@
   process.include_top_n:
     by_cpu: 5      # include top 5 processes by CPU
     by_memory: 5   # include top 5 processes by memory
-  # Configure the mount point of the host’s filesystem for use in monitoring a host from within a container
-  #system.hostfs: "/hostfs"
+# Configure the mount point of the host’s filesystem for use in monitoring a host from within a container
+#system.hostfs: "/hostfs"
 
 - module: system
   period: 1m

metricbeat/module/system/core/core.go

@@ -26,6 +26,7 @@ import (
 	metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/cpu"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+	"github.com/elastic/beats/v7/metricbeat/module/system"
 )
 
 func init() {
@@ -56,11 +57,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if config.CPUTicks != nil && *config.CPUTicks {
 		config.Metrics = append(config.Metrics, "ticks")
 	}
-
+	sys := base.Module().(system.SystemModule)
 	return &MetricSet{
 		BaseMetricSet: base,
 		opts:          opts,
-		cores:         metrics.New(""),
+		cores:         metrics.New(sys.GetHostFS()),
 	}, nil
 }
 

metricbeat/module/system/cpu/cpu.go

@@ -27,6 +27,7 @@ import (
 	metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/cpu"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+	"github.com/elastic/beats/v7/metricbeat/module/system"
 )
 
 func init() {
@@ -58,11 +59,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if config.CPUTicks != nil && *config.CPUTicks {
 		config.Metrics = append(config.Metrics, "ticks")
 	}
-
+	sys := base.Module().(system.SystemModule)
 	return &MetricSet{
 		BaseMetricSet: base,
 		opts:          opts,
-		cpu:           metrics.New(""),
+		cpu:           metrics.New(sys.GetHostFS()),
 	}, nil
 }
 

metricbeat/module/system/diskio/diskio.go

@@ -21,13 +21,10 @@
 package diskio
 
 import (
-	"os"
-	"path/filepath"
 	"runtime"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/metric/system/diskio"
-	"github.com/elastic/beats/v7/libbeat/paths"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
 
@@ -63,7 +60,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
-	os.Setenv("HOST_PROC", filepath.Join(paths.Paths.Hostfs, "/proc"))
 
 	return &MetricSet{
 		BaseMetricSet:  base,

metricbeat/module/system/entropy/entropy.go

@@ -23,15 +23,16 @@ package entropy
 import (
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
-	"github.com/elastic/beats/v7/libbeat/paths"
 	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/module/system"
 )
 
 // init registers the MetricSet with the central registry as soon as the program
@@ -56,7 +57,8 @@ type MetricSet struct {
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	cfgwarn.Beta("The system entropy metricset is beta.")
 
-	totalPath := paths.Resolve(paths.Hostfs, "/proc/sys/kernel/random")
+	sys := base.Module().(system.SystemModule)
+	totalPath := filepath.Join(sys.GetHostFS(), "/proc/sys/kernel/random")
 
 	return &MetricSet{
 		BaseMetricSet: base,

metricbeat/module/system/memory/memory.go

@@ -28,6 +28,7 @@ import (
 	metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/memory"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+	"github.com/elastic/beats/v7/metricbeat/module/system"
 )
 
 func init() {
@@ -40,17 +41,19 @@ func init() {
 // MetricSet for fetching system memory metrics.
 type MetricSet struct {
 	mb.BaseMetricSet
+	mod system.SystemModule
 }
 
 // New is a mb.MetricSetFactory that returns a memory.MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	return &MetricSet{BaseMetricSet: base}, nil
+	sys := base.Module().(system.SystemModule)
+	return &MetricSet{BaseMetricSet: base, mod: sys}, nil
 }
 
 // Fetch fetches memory metrics from the OS.
 func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 
-	eventRaw, err := metrics.Get("")
+	eventRaw, err := metrics.Get(m.mod.GetHostFS())
 	if err != nil {
 		return errors.Wrap(err, "error fetching memory metrics")
 	}