Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix used memory calculations in docker/memory #27473

Merged
merged 2 commits into from
Aug 23, 2021
+122 −43
Merged

Fix used memory calculations in docker/memory #27473

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2547223
fix used memory calculations in docker/memory
fearful-symmetry Aug 18, 2021
b1c1c62
Merge remote-tracking branch 'upstream/master' into docker-memory-sta…
fearful-symmetry Aug 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix used memory calculations in docker/memory
  • Loading branch information
@fearful-symmetry
fearful-symmetry committed Aug 18, 2021
commit 25472234412977f4cfeccd68e959007d052567e4
116 changes: 75 additions & 41 deletions metricbeat/module/docker/memory/_meta/data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,61 +1,94 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"container": {
"id": "aa41902101351f415e6e983b0673c0ba715dd4bc316bd5fc0ebd6fcf94287f86",
"id": "23ce6f1b53181ea3db0611fe4de36f0ebf1c0a37cb8272e028cac06240dafbe0",
"image": {
"name": "redis:latest"
"name": "docker.elastic.co/beats/elastic-agent:7.15.0-SNAPSHOT"
},
"name": "amazing_cohen",
"name": "elastic-package-stack_elastic-agent_1",
"runtime": "docker"
},
"docker": {
"container": {
"labels": {
"com_docker_compose_config-hash": "8e3d03827946685d53a2f171a126c397a3278da18ecd68a970cba9131160c52c",
"com_docker_compose_container-number": "1",
"com_docker_compose_oneoff": "False",
"com_docker_compose_project": "elastic-package-stack",
"com_docker_compose_service": "elastic-agent",
"com_docker_compose_version": "1.28.6",
"description": "Agent manages other beats based on configuration provided.",
"io_k8s_description": "Agent manages other beats based on configuration provided.",
"io_k8s_display-name": "Elastic-Agent image",
"license": "Elastic License",
"maintainer": "infra@elastic.co",
"name": "elastic-agent",
"org_label-schema_build-date": "2021-07-28T09:55:40Z",
"org_label-schema_license": "Elastic License",
"org_label-schema_name": "elastic-agent",
"org_label-schema_schema-version": "1.0",
"org_label-schema_url": "https://www.elastic.co/beats/elastic-agent",
"org_label-schema_vcs-ref": "16108a69f9f437c00cb6125c57bbc01c4eb805bb",
"org_label-schema_vcs-url": "github.com/elastic/beats/v7",
"org_label-schema_vendor": "Elastic",
"org_label-schema_version": "7.15.0-SNAPSHOT",
"org_opencontainers_image_created": "2021-07-28T09:55:40Z",
"org_opencontainers_image_licenses": "Elastic License",
"org_opencontainers_image_title": "Elastic-Agent",
"org_opencontainers_image_vendor": "Elastic",
"release": "1",
"summary": "elastic-agent",
"url": "https://www.elastic.co/beats/elastic-agent",
"vendor": "Elastic",
"version": "7.15.0-SNAPSHOT"
}
},
"memory": {
"fail": {
"count": 0
},
"limit": 2095878144,
"limit": 67514433536,
"rss": {
"pct": 0.0004025882909345325,
"total": 843776
"pct": 0,
"total": 0
},
"stats": {
"active_anon": 421888,
"active_file": 36864,
"cache": 86016,
"dirty": 0,
"hierarchical_memory_limit": 9223372036854771712,
"hierarchical_memsw_limit": 9223372036854771712,
"inactive_anon": 421888,
"inactive_file": 49152,
"mapped_file": 53248,
"pgfault": 1587,
"pgmajfault": 1,
"pgpgin": 2426,
"pgpgout": 2199,
"rss": 843776,
"rss_huge": 0,
"total_active_anon": 421888,
"total_active_file": 36864,
"total_cache": 86016,
"total_dirty": 0,
"total_inactive_anon": 421888,
"total_inactive_file": 49152,
"total_mapped_file": 53248,
"total_pgfault": 1587,
"total_pgmajfault": 1,
"total_pgpgin": 2426,
"total_pgpgout": 2199,
"total_rss": 843776,
"total_rss_huge": 0,
"total_unevictable": 0,
"total_writeback": 0,
"active_anon": 270336,
"active_file": 135168,
"anon": 246484992,
"anon_thp": 4194304,
"file": 325484544,
"file_dirty": 0,
"file_mapped": 170582016,
"file_writeback": 0,
"inactive_anon": 250257408,
"inactive_file": 325619712,
"kernel_stack": 2703360,
"pgactivate": 62898,
"pgdeactivate": 0,
"pgfault": 2150971515,
"pglazyfree": 207999,
"pglazyfreed": 0,
"pgmajfault": 0,
"pgrefill": 0,
"pgscan": 0,
"pgsteal": 0,
"shmem": 0,
"slab": 8112800,
"slab_reclaimable": 5753632,
"slab_unreclaimable": 2359168,
"sock": 200704,
"thp_collapse_alloc": 0,
"thp_fault_alloc": 0,
"unevictable": 0,
"writeback": 0
"workingset_activate": 0,
"workingset_nodereclaim": 0,
"workingset_refault": 0
},
"usage": {
"max": 7860224,
"pct": 0.000672283359618831,
"total": 1409024
"max": 0,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this 0 look right to you?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a case where V1 and V2 might be different. Max Just indicates the cgroup's upper threshold, with 0 meaning there is no upper limit.

"pct": 0.0039415723433138695,
"total": 266113024
}
}
},
Expand All @@ -65,7 +98,8 @@
"module": "docker"
},
"metricset": {
"name": "memory"
"name": "memory",
"period": 10000
},
"service": {
"address": "/var/run/docker.sock",
Expand Down
21 changes: 19 additions & 2 deletions metricbeat/module/docker/memory/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,23 @@ func (s *MemoryService) getMemoryStatsList(containers []docker.Stat, dedot bool)

func (s *MemoryService) getMemoryStats(myRawStat docker.Stat, dedot bool) MemoryData {
totalRSS := myRawStat.Stats.MemoryStats.Stats["total_rss"]

// Emulate newer docker releases and exclude cache values from memory usage
// See here for a little more context. usage - cache won't work, as it includes shared mappings that can't be dropped
// like regular cache.
// This formula is used by both cadvisor and containerd
// https://github.com/google/cadvisor/commit/307d1b1cb320fef66fab02db749f07a459245451
var fileUsage, memUsage uint64
// use this a shortcut to see if the underlying system is V1 or V2
totalInactive, isV1 := myRawStat.Stats.MemoryStats.Stats["total_inactive_file"]
if isV1 && totalInactive < myRawStat.Stats.MemoryStats.Usage {
fileUsage = totalInactive
}
if fileInactive, ok := myRawStat.Stats.MemoryStats.Stats["inactive_file"]; !isV1 && ok && fileInactive < myRawStat.Stats.MemoryStats.Usage {
fileUsage = fileInactive
}

memUsage = myRawStat.Stats.MemoryStats.Usage - fileUsage
return MemoryData{
Time: common.Time(myRawStat.Stats.Read),
Container: docker.NewContainer(myRawStat.Container, dedot),
Expand All @@ -70,8 +87,8 @@ func (s *MemoryService) getMemoryStats(myRawStat docker.Stat, dedot bool) Memory
MaxUsage: myRawStat.Stats.MemoryStats.MaxUsage,
TotalRss: totalRSS,
TotalRssP: float64(totalRSS) / float64(myRawStat.Stats.MemoryStats.Limit),
Usage: myRawStat.Stats.MemoryStats.Usage,
UsageP: float64(myRawStat.Stats.MemoryStats.Usage) / float64(myRawStat.Stats.MemoryStats.Limit),
Usage: memUsage,
UsageP: float64(memUsage) / float64(myRawStat.Stats.MemoryStats.Limit),
Stats: myRawStat.Stats.MemoryStats.Stats,
//Windows memory statistics
Commit: myRawStat.Stats.MemoryStats.Commit,
Expand Down
28 changes: 28 additions & 0 deletions metricbeat/module/docker/memory/memory_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,34 @@ func TestMemoryServiceBadData(t *testing.T) {

}

func TestMemoryMath(t *testing.T) {
memStats := types.StatsJSON{
Stats: types.Stats{
Read: time.Now(),
PreCPUStats: types.CPUStats{
CPUUsage: types.CPUUsage{
TotalUsage: 200,
},
},
MemoryStats: types.MemoryStats{
Limit: 5,
Usage: 5000,
Stats: map[string]uint64{
"total_inactive_file": 1000, // CGV1
"inactive_file": 900,
},
}, //Test for cases where this is empty
},
}

memoryService := &MemoryService{}
memoryRawStats := []docker.Stat{
docker.Stat{Stats: memStats, Container: &types.Container{Names: []string{"test-container"}, Labels: map[string]string{}}},
}
rawStats := memoryService.getMemoryStatsList(memoryRawStats, false)
assert.Equal(t, float64(800), rawStats[0].UsageP) // 5000-900 /5
}

func getMemoryStats(read time.Time, number uint64) types.StatsJSON {

myMemoryStats := types.StatsJSON{
Expand Down