Report container FS metrics into prometheus /metrics

PerDiskStats reported from cgroups were not being surfaced into prometheus. In order to properly correlate the metrics, we need to assign a device label to each metric (which is the FS or device path). Since blkio cgroup tracks devices, we create a synthetic device `/dev/NAME` for the metric. Assign a Device label to each PerDiskStat for the handlers up front, and then surface the PerDiskStat values into the prometheus metrics. Report two new metrics - total bytes read and total bytes written.
mannychang · Apr 25, 2017 · 4e25a79 · 4e25a79
1 parent 0073fcb
commit 4e25a79
Show file tree

Hide file tree

Showing 8 changed files with 230 additions and 43 deletions.
diff --git a/container/common/helpers.go b/container/common/helpers.go
@@ -223,3 +223,71 @@ func ListContainers(name string, cgroupPaths map[string]string, listType contain
 
 	return ret, nil
 }
+
+// AssignDeviceNamesToDiskStats assigns the Device field on the provided DiskIoStats by looking up
+// the device major and minor identifiers in the provided device namer.
+func AssignDeviceNamesToDiskStats(namer DeviceNamer, stats *info.DiskIoStats) {
+	assignDeviceNamesToPerDiskStats(
+		namer,
+		stats.IoMerged,
+		stats.IoQueued,
+		stats.IoServiceBytes,
+		stats.IoServiceTime,
+		stats.IoServiced,
+		stats.IoTime,
+		stats.IoWaitTime,
+		stats.Sectors,
+	)
+}
+
+// assignDeviceNamesToPerDiskStats looks up device names for the provided stats, caching names
+// if necessary.
+func assignDeviceNamesToPerDiskStats(namer DeviceNamer, diskStats ...[]info.PerDiskStats) {
+	devices := make(deviceIdentifierMap)
+	for _, stats := range diskStats {
+		for i, stat := range stats {
+			stats[i].Device = devices.Find(stat.Major, stat.Minor, namer)
+		}
+	}
+}
+
+// DeviceNamer returns string names for devices by their major and minor id.
+type DeviceNamer interface {
+	// DeviceName returns the name of the device by its major and minor ids, or false if no
+	// such device is recognized.
+	DeviceName(major, minor uint64) (string, bool)
+}
+
+type MachineInfoNamer info.MachineInfo
+
+func (n *MachineInfoNamer) DeviceName(major, minor uint64) (string, bool) {
+	for _, info := range n.DiskMap {
+		if info.Major == major && info.Minor == minor {
+			return "/dev/" + info.Name, true
+		}
+	}
+	for _, info := range n.Filesystems {
+		if info.DeviceMajor == major && info.DeviceMinor == minor {
+			return info.Device, true
+		}
+	}
+	return "", false
+}
+
+type deviceIdentifier struct {
+	major uint64
+	minor uint64
+}
+
+type deviceIdentifierMap map[deviceIdentifier]string
+
+// Find locates the device name by device identifier out of from, caching the result as necessary.
+func (m deviceIdentifierMap) Find(major, minor uint64, namer DeviceNamer) string {
+	d := deviceIdentifier{major, minor}
+	if s, ok := m[d]; ok {
+		return s
+	}
+	s, _ := namer.DeviceName(major, minor)
+	m[d] = s
+	return s
+}
diff --git a/container/docker/handler.go b/container/docker/handler.go
@@ -390,6 +390,15 @@ func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) {
 }
 
 func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error {
+	mi, err := self.machineInfoFactory.GetMachineInfo()
+	if err != nil {
+		return err
+	}
+
+	if !self.ignoreMetrics.Has(container.DiskIOMetrics) {
+		common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo)
+	}
+
 	if self.ignoreMetrics.Has(container.DiskUsageMetrics) {
 		return nil
 	}
@@ -411,11 +420,6 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error
 		return nil
 	}
 
-	mi, err := self.machineInfoFactory.GetMachineInfo()
-	if err != nil {
-		return err
-	}
-
 	var (
 		limit  uint64
 		fsType string

diff --git a/container/raw/handler.go b/container/raw/handler.go
@@ -197,6 +197,7 @@ func fsToFsStats(fs *fs.Fs) info.FsStats {
 }
 
 func (self *rawContainerHandler) getFsStats(stats *info.ContainerStats) error {
+	var allFs []fs.Fs
 	// Get Filesystem information only for the root cgroup.
 	if isRootCgroup(self.name) {
 		filesystems, err := self.fsInfo.GetGlobalFsInfo()
@@ -207,6 +208,7 @@ func (self *rawContainerHandler) getFsStats(stats *info.ContainerStats) error {
 			fs := filesystems[i]
 			stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs))
 		}
+		allFs = filesystems
 	} else if len(self.externalMounts) > 0 {
 		var mountSet map[string]struct{}
 		mountSet = make(map[string]struct{})
@@ -221,7 +223,10 @@ func (self *rawContainerHandler) getFsStats(stats *info.ContainerStats) error {
 			fs := filesystems[i]
 			stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs))
 		}
+		allFs = filesystems
 	}
+
+	common.AssignDeviceNamesToDiskStats(&fsNamer{fs: allFs, factory: self.machineInfoFactory}, &stats.DiskIo)
 	return nil
 }
 
@@ -272,3 +277,25 @@ func (self *rawContainerHandler) Exists() bool {
 func (self *rawContainerHandler) Type() container.ContainerType {
 	return container.ContainerTypeRaw
 }
+
+type fsNamer struct {
+	fs      []fs.Fs
+	factory info.MachineInfoFactory
+	info    common.DeviceNamer
+}
+
+func (n *fsNamer) DeviceName(major, minor uint64) (string, bool) {
+	for _, info := range n.fs {
+		if uint64(info.Major) == major && uint64(info.Minor) == minor {
+			return info.Device, true
+		}
+	}
+	if n.info == nil {
+		mi, err := n.factory.GetMachineInfo()
+		if err != nil {
+			return "", false
+		}
+		n.info = (*common.MachineInfoNamer)(mi)
+	}
+	return n.info.DeviceName(major, minor)
+}
diff --git a/container/rkt/handler.go b/container/rkt/handler.go
@@ -202,6 +202,15 @@ func (handler *rktContainerHandler) GetSpec() (info.ContainerSpec, error) {
 }
 
 func (handler *rktContainerHandler) getFsStats(stats *info.ContainerStats) error {
+	mi, err := handler.machineInfoFactory.GetMachineInfo()
+	if err != nil {
+		return err
+	}
+
+	if !handler.ignoreMetrics.Has(container.DiskIOMetrics) {
+		common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo)
+	}
+
 	if handler.ignoreMetrics.Has(container.DiskUsageMetrics) {
 		return nil
 	}
@@ -211,10 +220,6 @@ func (handler *rktContainerHandler) getFsStats(stats *info.ContainerStats) error
 		return err
 	}
 
-	mi, err := handler.machineInfoFactory.GetMachineInfo()
-	if err != nil {
-		return err
-	}
 	var limit uint64 = 0
 
 	// Use capacity as limit.

diff --git a/info/v1/container.go b/info/v1/container.go
@@ -307,9 +307,10 @@ type CpuStats struct {
 }
 
 type PerDiskStats struct {
-	Major uint64            `json:"major"`
-	Minor uint64            `json:"minor"`
-	Stats map[string]uint64 `json:"stats"`
+	Device string            `json:"-"`
+	Major  uint64            `json:"major"`
+	Minor  uint64            `json:"minor"`
+	Stats  map[string]uint64 `json:"stats"`
 }
 
 type DiskIoStats struct {

diff --git a/info/v1/machine.go b/info/v1/machine.go
@@ -17,6 +17,10 @@ package v1
 type FsInfo struct {
 	// Block device associated with the filesystem.
 	Device string `json:"device"`
+	// DeviceMajor is the major identifier of the device, used for correlation with blkio stats
+	DeviceMajor uint64 `json:"-"`
+	// DeviceMinor is the minor identifier of the device, used for correlation with blkio stats
+	DeviceMinor uint64 `json:"-"`
 
 	// Total number of bytes available on the filesystem.
 	Capacity uint64 `json:"capacity"`

diff --git a/machine/info.go b/machine/info.go
@@ -116,7 +116,7 @@ func Info(sysFs sysfs.SysFs, fsInfo fs.FsInfo, inHostNamespace bool) (*info.Mach
 		if fs.Inodes != nil {
 			inodes = *fs.Inodes
 		}
-		machineInfo.Filesystems = append(machineInfo.Filesystems, info.FsInfo{Device: fs.Device, Type: fs.Type.String(), Capacity: fs.Capacity, Inodes: inodes, HasInodes: fs.Inodes != nil})
+		machineInfo.Filesystems = append(machineInfo.Filesystems, info.FsInfo{Device: fs.Device, DeviceMajor: uint64(fs.Major), DeviceMinor: uint64(fs.Minor), Type: fs.Type.String(), Capacity: fs.Capacity, Inodes: inodes, HasInodes: fs.Inodes != nil})
 	}
 
 	return machineInfo, nil