feat: new input Hugepages plugin (#10763)

plugins/inputs/all/all.go

@@ -67,6 +67,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/http_listener_v2"
 	_ "github.com/influxdata/telegraf/plugins/inputs/http_response"
 	_ "github.com/influxdata/telegraf/plugins/inputs/httpjson"
+	_ "github.com/influxdata/telegraf/plugins/inputs/hugepages"
 	_ "github.com/influxdata/telegraf/plugins/inputs/icinga2"
 	_ "github.com/influxdata/telegraf/plugins/inputs/infiniband"
 	_ "github.com/influxdata/telegraf/plugins/inputs/influxdb"

plugins/inputs/hugepages/README.md

@@ -0,0 +1,67 @@
+# Hugepages Input Plugin
+
+Transparent Huge Pages (THP) is a Linux memory management system that reduces the overhead of
+Translation Lookaside Buffer (TLB) lookups on machines with large amounts of memory by using larger
+memory pages.
+
+Consult <https://www.kernel.org/doc/html/latest/admin-guide/mm/hugetlbpage.html> for more details.
+
+## Configuration
+
+```toml
+# Gathers huge pages measurements.
+[[inputs.hugepages]]
+  ## Supported huge page types:
+  ##   - "root" - based on root huge page control directory: /sys/kernel/mm/hugepages
+  ##   - "per_node" - based on per NUMA node directories: /sys/devices/system/node/node[0-9]*/hugepages
+  ##   - "meminfo" - based on /proc/meminfo file
+  # types = ["root", "per_node"]
+```
+
+## Measurements
+
+**The following measurements are supported by Hugepages plugin:**
+
+- hugepages_root (gathered from root huge page control directory: `/sys/kernel/mm/hugepages`)
+  - tags:
+    - size_kb (integer, kB)
+  - fields:
+    - free (integer)
+    - mempolicy (integer)
+    - overcommit (integer)
+    - reserved (integer)
+    - surplus (integer)
+    - total (integer)
+- hugepages_per_node (gathered from per NUMA node directories: `/sys/devices/system/node/node[0-9]*/hugepages`)
+  - tags:
+    - size_kb (integer, kB)
+    - node (integer)
+  - fields:
+    - free (integer)
+    - surplus (integer)
+    - total (integer)
+- hugepages_meminfo (gathered from `/proc/meminfo` file)
+  - The fields `total`, `free`, `reserved`, and `surplus` are counts of pages of default size. Fields with suffix `_kb` are in kilobytes.
+  - fields:
+    - anonymous_kb (integer, kB)
+    - file_kb (integer, kB)
+    - free (integer)
+    - reserved (integer)
+    - shared_kb (integer, kB)
+    - size_kb (integer, kB)
+    - surplus (integer)
+    - tlb_kb (integer, kB)
+    - total (integer)
+
+## Example Output
+
+```text
+$ ./telegraf -config telegraf.conf -input-filter hugepages -test
+> hugepages_root,host=ubuntu,size_kb=1048576 free=0i,mempolicy=8i,overcommit=0i,reserved=0i,surplus=0i,total=8i 1646258020000000000
+> hugepages_root,host=ubuntu,size_kb=2048 free=883i,mempolicy=2048i,overcommit=0i,reserved=0i,surplus=0i,total=2048i 1646258020000000000
+> hugepages_per_node,host=ubuntu,size_kb=1048576,node=0 free=0i,surplus=0i,total=4i 1646258020000000000
+> hugepages_per_node,host=ubuntu,size_kb=2048,node=0 free=434i,surplus=0i,total=1024i 1646258020000000000
+> hugepages_per_node,host=ubuntu,size_kb=1048576,node=1 free=0i,surplus=0i,total=4i 1646258020000000000
+> hugepages_per_node,host=ubuntu,size_kb=2048,node=1 free=449i,surplus=0i,total=1024i 1646258020000000000
+> hugepages_meminfo,host=ubuntu anonymous_kb=0i,file_kb=0i,free=883i,reserved=0i,shared_kb=0i,size_kb=2048i,surplus=0i,tlb_kb=12582912i,total=2048i 1646258020000000000
+```

plugins/inputs/hugepages/hugepages.go

@@ -0,0 +1,287 @@
+//go:build linux
+// +build linux
+
+package hugepages
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+const (
+	// path to root huge page control directory
+	rootHugepagePath = "/sys/kernel/mm/hugepages"
+	// path where per NUMA node statistics are kept
+	numaNodePath = "/sys/devices/system/node"
+	// path to the meminfo file
+	meminfoPath = "/proc/meminfo"
+
+	rootHugepages    = "root"
+	perNodeHugepages = "per_node"
+	meminfoHugepages = "meminfo"
+
+	hugepagesSampleConfig = `
+  ## Supported huge page types:
+  ##   - "root" - based on root huge page control directory: /sys/kernel/mm/hugepages
+  ##   - "per_node" - based on per NUMA node directories: /sys/devices/system/node/node[0-9]*/hugepages
+  ##   - "meminfo" - based on /proc/meminfo file
+  # types = ["root", "per_node"]
+`
+)
+
+var (
+	newlineByte = []byte("\n")
+	colonByte   = []byte(":")
+
+	hugepagesMetricsRoot = map[string]string{
+		"free_hugepages":          "free",
+		"nr_hugepages":            "total",
+		"nr_hugepages_mempolicy":  "mempolicy",
+		"nr_overcommit_hugepages": "overcommit",
+		"resv_hugepages":          "reserved",
+		"surplus_hugepages":       "surplus",
+	}
+
+	hugepagesMetricsPerNUMANode = map[string]string{
+		"free_hugepages":    "free",
+		"nr_hugepages":      "total",
+		"surplus_hugepages": "surplus",
+	}
+
+	hugepagesMetricsFromMeminfo = map[string]string{
+		"HugePages_Total": "total",
+		"HugePages_Free":  "free",
+		"HugePages_Rsvd":  "reserved",
+		"HugePages_Surp":  "surplus",
+		"Hugepagesize":    "size_kb",
+		"Hugetlb":         "tlb_kb",
+		"AnonHugePages":   "anonymous_kb",
+		"ShmemHugePages":  "shared_kb",
+		"FileHugePages":   "file_kb",
+	}
+)
+
+type Hugepages struct {
+	Types []string `toml:"types"`
+
+	gatherRoot    bool
+	gatherPerNode bool
+	gatherMeminfo bool
+
+	rootHugepagePath string
+	numaNodePath     string
+	meminfoPath      string
+}
+
+func (h *Hugepages) Description() string {
+	return "Gathers huge pages measurements."
+}
+
+func (h *Hugepages) SampleConfig() string {
+	return hugepagesSampleConfig
+}
+
+func (h *Hugepages) Init() error {
+	err := h.parseHugepagesConfig()
+	if err != nil {
+		return err
+	}
+
+	h.rootHugepagePath = rootHugepagePath
+	h.numaNodePath = numaNodePath
+	h.meminfoPath = meminfoPath
+
+	return nil
+}
+
+func (h *Hugepages) Gather(acc telegraf.Accumulator) error {
+	if h.gatherRoot {
+		if err := h.gatherRootStats(acc); err != nil {
+			return fmt.Errorf("gathering root stats failed: %v", err)
+		}
+	}
+
+	if h.gatherPerNode {
+		if err := h.gatherStatsPerNode(acc); err != nil {
+			return fmt.Errorf("gathering per node stats failed: %v", err)
+		}
+	}
+
+	if h.gatherMeminfo {
+		if err := h.gatherStatsFromMeminfo(acc); err != nil {
+			return fmt.Errorf("gathering meminfo stats failed: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// gatherStatsPerNode collects root hugepages statistics
+func (h *Hugepages) gatherRootStats(acc telegraf.Accumulator) error {
+	return h.gatherFromHugepagePath(acc, "hugepages_"+rootHugepages, h.rootHugepagePath, hugepagesMetricsRoot, nil)
+}
+
+// gatherStatsPerNode collects hugepages statistics per NUMA node
+func (h *Hugepages) gatherStatsPerNode(acc telegraf.Accumulator) error {
+	nodeDirs, err := ioutil.ReadDir(h.numaNodePath)
+	if err != nil {
+		return err
+	}
+
+	// read metrics from: node*/hugepages/hugepages-*/*
+	for _, nodeDir := range nodeDirs {
+		if !nodeDir.IsDir() || !strings.HasPrefix(nodeDir.Name(), "node") {
+			continue
+		}
+
+		nodeNumber := strings.TrimPrefix(nodeDir.Name(), "node")
+		_, err := strconv.Atoi(nodeNumber)
+		if err != nil {
+			continue
+		}
+
+		perNodeTags := map[string]string{
+			"node": nodeNumber,
+		}
+		hugepagesPath := filepath.Join(h.numaNodePath, nodeDir.Name(), "hugepages")
+		err = h.gatherFromHugepagePath(acc, "hugepages_"+perNodeHugepages, hugepagesPath, hugepagesMetricsPerNUMANode, perNodeTags)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (h *Hugepages) gatherFromHugepagePath(acc telegraf.Accumulator, measurement, path string, fileFilter map[string]string, defaultTags map[string]string) error {
+	// read metrics from: hugepages/hugepages-*/*
+	hugepagesDirs, err := ioutil.ReadDir(path)
+	if err != nil {
+		return fmt.Errorf("reading root dir failed: %v", err)
+	}
+
+	for _, hugepagesDir := range hugepagesDirs {
+		if !hugepagesDir.IsDir() || !strings.HasPrefix(hugepagesDir.Name(), "hugepages-") {
+			continue
+		}
+
+		hugepagesSize := strings.TrimPrefix(strings.TrimSuffix(hugepagesDir.Name(), "kB"), "hugepages-")
+		_, err := strconv.Atoi(hugepagesSize)
+		if err != nil {
+			continue
+		}
+
+		metricsPath := filepath.Join(path, hugepagesDir.Name())
+		metricFiles, err := ioutil.ReadDir(metricsPath)
+		if err != nil {
+			return fmt.Errorf("reading metric dir failed: %v", err)
+		}
+
+		metrics := make(map[string]interface{})
+		for _, metricFile := range metricFiles {
+			metricName, ok := fileFilter[metricFile.Name()]
+			if mode := metricFile.Mode(); !mode.IsRegular() || !ok {
+				continue
+			}
+
+			metricFullPath := filepath.Join(metricsPath, metricFile.Name())
+			metricBytes, err := ioutil.ReadFile(metricFullPath)
+			if err != nil {
+				return err
+			}
+
+			metricValue, err := strconv.Atoi(string(bytes.TrimSuffix(metricBytes, newlineByte)))
+			if err != nil {
+				return fmt.Errorf("failed to convert content of '%s': %v", metricFullPath, err)
+			}
+
+			metrics[metricName] = metricValue
+		}
+
+		if len(metrics) == 0 {
+			continue
+		}
+
+		tags := make(map[string]string)
+		for key, value := range defaultTags {
+			tags[key] = value
+		}
+		tags["size_kb"] = hugepagesSize
+
+		acc.AddFields(measurement, metrics, tags)
+	}
+	return nil
+}
+
+// gatherStatsFromMeminfo collects hugepages statistics from meminfo file
+func (h *Hugepages) gatherStatsFromMeminfo(acc telegraf.Accumulator) error {
+	meminfo, err := ioutil.ReadFile(h.meminfoPath)
+	if err != nil {
+		return err
+	}
+
+	metrics := make(map[string]interface{})
+	lines := bytes.Split(meminfo, newlineByte)
+	for _, line := range lines {
+		fields := bytes.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		fieldName := string(bytes.TrimSuffix(fields[0], colonByte))
+		metricName, ok := hugepagesMetricsFromMeminfo[fieldName]
+		if !ok {
+			continue
+		}
+
+		fieldValue, err := strconv.Atoi(string(fields[1]))
+		if err != nil {
+			return fmt.Errorf("failed to convert content of '%s': %v", fieldName, err)
+		}
+
+		metrics[metricName] = fieldValue
+	}
+
+	acc.AddFields("hugepages_"+meminfoHugepages, metrics, map[string]string{})
+	return nil
+}
+
+func (h *Hugepages) parseHugepagesConfig() error {
+	// default
+	if h.Types == nil {
+		h.gatherRoot = true
+		h.gatherMeminfo = true
+		return nil
+	}
+
+	// empty array
+	if len(h.Types) == 0 {
+		return fmt.Errorf("plugin was configured with nothing to read")
+	}
+
+	for _, hugepagesType := range h.Types {
+		switch hugepagesType {
+		case rootHugepages:
+			h.gatherRoot = true
+		case perNodeHugepages:
+			h.gatherPerNode = true
+		case meminfoHugepages:
+			h.gatherMeminfo = true
+		default:
+			return fmt.Errorf("provided hugepages type `%s` is not valid", hugepagesType)
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	inputs.Add("hugepages", func() telegraf.Input {
+		return &Hugepages{}
+	})
+}

plugins/inputs/hugepages/hugepages_notlinux.go

@@ -0,0 +1,4 @@
+//go:build !linux
+// +build !linux
+
+package hugepages