Merge pull request #567 from Mashimiao/support-hugetlb-set-and-getstats

hugetlb: Add support of Set and GetStats function

cgroups/fs/apply_raw.go

@@ -30,6 +30,7 @@ var (
 		"freezer":    &FreezerGroup{},
 	}
 	CgroupProcesses = "cgroup.procs"
+	HugePageSizes, _ = cgroups.GetHugePageSize()
 )
 
 type subsystem interface {

cgroups/fs/hugetlb.go

@@ -3,6 +3,10 @@
 package fs
 
 import (
+	"fmt"
+	"strconv"
+	"strings"
+
 	"github.com/docker/libcontainer/cgroups"
 	"github.com/docker/libcontainer/configs"
 )
@@ -11,14 +15,25 @@ type HugetlbGroup struct {
 }
 
 func (s *HugetlbGroup) Apply(d *data) error {
-	// we just want to join this group even though we don't set anything
-	if _, err := d.join("hugetlb"); err != nil && !cgroups.IsNotFound(err) {
+	dir, err := d.join("hugetlb")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+
+	if err := s.Set(dir, d.c); err != nil {
 		return err
 	}
+
 	return nil
 }
 
 func (s *HugetlbGroup) Set(path string, cgroup *configs.Cgroup) error {
+	for _, hugetlb := range cgroup.HugetlbLimit {
+		if err := writeFile(path, strings.Join([]string{"hugetlb", hugetlb.Pagesize, "limit_in_bytes"}, "."), strconv.Itoa(hugetlb.Limit)); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -27,5 +42,31 @@ func (s *HugetlbGroup) Remove(d *data) error {
 }
 
 func (s *HugetlbGroup) GetStats(path string, stats *cgroups.Stats) error {
+	hugetlbStats := cgroups.HugetlbStats{}
+	for _, pageSize := range HugePageSizes {
+		usage := strings.Join([]string{"hugetlb", pageSize, "usage_in_bytes"}, ".")
+		value, err := getCgroupParamUint(path, usage)
+		if err != nil {
+			return fmt.Errorf("failed to parse %s - %v", usage, err)
+		}
+		hugetlbStats.Usage = value
+
+		maxUsage := strings.Join([]string{"hugetlb", pageSize, "max_usage_in_bytes"}, ".")
+		value, err = getCgroupParamUint(path, maxUsage)
+		if err != nil {
+			return fmt.Errorf("failed to parse %s - %v", maxUsage, err)
+		}
+		hugetlbStats.MaxUsage = value
+
+		failcnt := strings.Join([]string{"hugetlb", pageSize, "failcnt"}, ".")
+		value, err = getCgroupParamUint(path, failcnt)
+		if err != nil {
+			return fmt.Errorf("failed to parse %s - %v", failcnt, err)
+		}
+		hugetlbStats.Failcnt = value
+
+		stats.HugetlbStats[pageSize] = hugetlbStats
+	}
+
 	return nil
 }

cgroups/fs/hugetlb_test.go

@@ -0,0 +1,138 @@
+package fs
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/docker/libcontainer/cgroups"
+	"github.com/docker/libcontainer/configs"
+)
+
+const (
+	hugetlbUsageContents    = "128\n"
+	hugetlbMaxUsageContents = "256\n"
+	hugetlbFailcnt          = "100\n"
+)
+
+var (
+	hugePageSize, _ = cgroups.GetHugePageSize()
+	usage           = strings.Join([]string{"hugetlb", hugePageSize[0], "usage_in_bytes"}, ".")
+	limit           = strings.Join([]string{"hugetlb", hugePageSize[0], "limit_in_bytes"}, ".")
+	maxUsage        = strings.Join([]string{"hugetlb", hugePageSize[0], "max_usage_in_bytes"}, ".")
+	failcnt         = strings.Join([]string{"hugetlb", hugePageSize[0], "failcnt"}, ".")
+)
+
+func TestHugetlbSetHugetlb(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+
+	const (
+		hugetlbBefore = 256
+		hugetlbAfter  = 512
+	)
+
+	helper.writeFileContents(map[string]string{
+		limit: strconv.Itoa(hugetlbBefore),
+	})
+
+	helper.CgroupData.c.HugetlbLimit = []*configs.HugepageLimit{
+		{
+			Pagesize: hugePageSize[0],
+			Limit:    hugetlbAfter,
+		},
+	}
+	hugetlb := &HugetlbGroup{}
+	if err := hugetlb.Set(helper.CgroupPath, helper.CgroupData.c); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, limit)
+	if err != nil {
+		t.Fatalf("Failed to parse %s - %s", limit, err)
+	}
+	if value != hugetlbAfter {
+		t.Fatalf("Set hugetlb.limit_in_bytes failed. Expected: %v, Got: %v", hugetlbAfter, value)
+	}
+}
+
+func TestHugetlbStats(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		usage:    hugetlbUsageContents,
+		maxUsage: hugetlbMaxUsageContents,
+		failcnt:  hugetlbFailcnt,
+	})
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedStats := cgroups.HugetlbStats{Usage: 128, MaxUsage: 256, Failcnt: 100}
+	expectHugetlbStatEquals(t, expectedStats, actualStats.HugetlbStats[hugePageSize[0]])
+}
+
+func TestHugetlbStatsNoUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		maxUsage: hugetlbMaxUsageContents,
+	})
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestHugetlbStatsNoMaxUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		usage: hugetlbUsageContents,
+	})
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestHugetlbStatsBadUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		usage:    "bad",
+		maxUsage: hugetlbMaxUsageContents,
+	})
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestHugetlbStatsBadMaxUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		usage:    hugetlbUsageContents,
+		maxUsage: "bad",
+	})
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}

cgroups/fs/stats_util_test.go

@@ -72,6 +72,13 @@ func expectThrottlingDataEquals(t *testing.T, expected, actual cgroups.Throttlin
 	}
 }
 
+func expectHugetlbStatEquals(t *testing.T, expected, actual cgroups.HugetlbStats) {
+	if expected != actual {
+		logrus.Printf("Expected hugetlb stats %v but found %v\n", expected, actual)
+		t.Fail()
+	}
+}
+
 func expectMemoryStatEquals(t *testing.T, expected, actual cgroups.MemoryStats) {
 	expectMemoryDataEquals(t, expected.Usage, actual.Usage)
 	expectMemoryDataEquals(t, expected.SwapUsage, actual.SwapUsage)

cgroups/stats.go

@@ -68,13 +68,25 @@ type BlkioStats struct {
 	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive,omitempty"`
 }
 
+type HugetlbStats struct {
+	// current res_counter usage for hugetlb
+	Usage uint64 `json:"usage,omitempty"`
+	// maximum usage ever recorded.
+	MaxUsage uint64 `json:"max_usage,omitempty"`
+	// number of times htgetlb usage allocation failure.
+	Failcnt uint64 `json:"failcnt"`
+}
+
 type Stats struct {
 	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
 	MemoryStats MemoryStats `json:"memory_stats,omitempty"`
 	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
+	// the map is in the format "size of hugepage: stats of the hugepage"
+	HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
 }
 
 func NewStats() *Stats {
 	memoryStats := MemoryStats{Stats: make(map[string]uint64)}
-	return &Stats{MemoryStats: memoryStats}
+	hugetlbStats := make(map[string]HugetlbStats)
+	return &Stats{MemoryStats: memoryStats, HugetlbStats: hugetlbStats}
 }

cgroups/systemd/apply_systemd.go

@@ -223,6 +223,9 @@ func (m *Manager) Apply(pid int) error {
 		return err
 	}
 
+	if err := joinHugetlb(c, pid); err != nil {
+		return err
+	}
 	// FIXME: Systemd does have `BlockIODeviceWeight` property, but we got problem
 	// using that (at least on systemd 208, see https://github.com/docker/libcontainer/pull/354),
 	// so use fs work around for now.
@@ -538,3 +541,13 @@ func joinBlkio(c *configs.Cgroup, pid int) error {
 
 	return nil
 }
+
+func joinHugetlb(c *configs.Cgroup, pid int) error {
+	path, err := join(c, "hugetlb", pid)
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+
+	hugetlb := subsystems["hugetlb"]
+	return hugetlb.Set(path, c)
+}

cgroups/utils.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/docker/pkg/units"
 )
 
 // https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt
@@ -238,3 +239,23 @@ func RemovePaths(paths map[string]string) (err error) {
 	}
 	return fmt.Errorf("Failed to remove paths: %s", paths)
 }
+
+func GetHugePageSize() ([]string, error) {
+	var pageSizes []string
+	sizeList := []string{"B", "kB", "MB", "GB", "TB", "PB"}
+	files, err := ioutil.ReadDir("/sys/kernel/mm/hugepages")
+	if err != nil {
+		return pageSizes, err
+	}
+	for _, st := range files {
+		nameArray := strings.Split(st.Name(), "-")
+		pageSize, err := units.RAMInBytes(nameArray[1])
+		if err != nil {
+			return []string{}, err
+		}
+		sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, sizeList)
+		pageSizes = append(pageSizes, sizeString)
+	}
+
+	return pageSizes, nil
+}

configs/cgroup.go

@@ -78,6 +78,9 @@ type Cgroup struct {
 	// set the freeze value for the process
 	Freezer FreezerState `json:"freezer"`
 
+	// Hugetlb limit (in bytes)
+	HugetlbLimit []*HugepageLimit `json:"hugetlb_limit"`
+
 	// Parent slice to use for systemd TODO: remove in favor or parent
 	Slice string `json:"slice"`
 

configs/hugepage_limit.go

@@ -0,0 +1,9 @@
+package configs
+
+type HugepageLimit struct {
+	// which type of hugepage to limit.
+	Pagesize string `json:"page_size"`
+
+	// usage limit for hugepage.
+	Limit int `json:"limit"`
+}