Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add device label to paging scraper. #4854

Merged
merged 4 commits into from
Oct 7, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pagingscraper

type pageFileStats struct {
deviceName string // Optional
usedBytes uint64
freeBytes uint64
cachedBytes *uint64 // Optional
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build linux
// +build linux

package pagingscraper

import (
"bufio"
"fmt"
"io"
"os"
"strconv"
"strings"
)

const swapsFilePath = "/proc/swaps"

// swaps file column indexes
const (
nameCol = 0
// typeCol = 1
totalCol = 2
usedCol = 3
// priorityCol = 4

minimumColCount = usedCol + 1
)

func getPageFileStats() ([]*pageFileStats, error) {
f, err := os.Open(swapsFilePath)
if err != nil {
return nil, err
}
defer f.Close()

return parseSwapsFile(f)
}

func parseSwapsFile(r io.Reader) ([]*pageFileStats, error) {
scanner := bufio.NewScanner(r)
if !scanner.Scan() {
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("couldn't read file %q: %w", swapsFilePath, err)
}
return nil, fmt.Errorf("unexpected end-of-file in %q", swapsFilePath)

}

// Check header headerFields are as expected
headerFields := strings.Fields(scanner.Text())
if len(headerFields) < minimumColCount {
return nil, fmt.Errorf("couldn't parse %q: expected ≥%d fields in header but got %v", swapsFilePath, minimumColCount, headerFields)
}
if headerFields[nameCol] != "Filename" {
return nil, fmt.Errorf("couldn't parse %q: expected %q to be %q", swapsFilePath, headerFields[nameCol], "Filename")
}
if headerFields[totalCol] != "Size" {
return nil, fmt.Errorf("couldn't parse %q: expected %q to be %q", swapsFilePath, headerFields[totalCol], "Size")
}
if headerFields[usedCol] != "Used" {
return nil, fmt.Errorf("couldn't parse %q: expected %q to be %q", swapsFilePath, headerFields[usedCol], "Used")
}

var swapDevices []*pageFileStats
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < minimumColCount {
return nil, fmt.Errorf("couldn't parse %q: expected ≥%d fields in row but got %v", swapsFilePath, minimumColCount, fields)
}

totalKiB, err := strconv.ParseUint(fields[totalCol], 10, 64)
if err != nil {
return nil, fmt.Errorf("couldn't parse 'Size' column in %q: %w", swapsFilePath, err)
}

usedKiB, err := strconv.ParseUint(fields[usedCol], 10, 64)
if err != nil {
return nil, fmt.Errorf("couldn't parse 'Used' column in %q: %w", swapsFilePath, err)
}

swapDevices = append(swapDevices, &pageFileStats{
deviceName: fields[nameCol],
usedBytes: usedKiB * 1024,
freeBytes: (totalKiB - usedKiB) * 1024,
})
}

if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("couldn't read file %q: %w", swapsFilePath, err)
}

return swapDevices, nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build linux
// +build linux

package pagingscraper

import (
"strings"
"testing"

"github.com/stretchr/testify/assert"
)

const validFile = `Filename Type Size Used Priority
/dev/dm-2 partition 67022844 490788 -2
/swapfile file 2 1 -3
`

const invalidFile = `INVALID Type Size Used Priority
/dev/dm-2 partition 67022844 490788 -2
/swapfile file 1048572 0 -3
`

func TestGetPageFileStats_ValidFile(t *testing.T) {
assert := assert.New(t)
stats, err := parseSwapsFile(strings.NewReader(validFile))
assert.NoError(err)

assert.Equal(*stats[0], pageFileStats{
deviceName: "/dev/dm-2",
usedBytes: 502566912,
freeBytes: 68128825344,
})

assert.Equal(*stats[1], pageFileStats{
deviceName: "/swapfile",
usedBytes: 1024,
freeBytes: 1024,
})
}

func TestGetPageFileStats_InvalidFile(t *testing.T) {
_, err := parseSwapsFile(strings.NewReader(invalidFile))
assert.Error(t, err)
}

func TestGetPageFileStats_EmptyFile(t *testing.T) {
_, err := parseSwapsFile(strings.NewReader(""))
assert.Error(t, err)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build !windows && !linux
// +build !windows,!linux

package pagingscraper

import "github.com/shirou/gopsutil/mem"

func getPageFileStats() ([]*pageFileStats, error) {
vmem, err := mem.VirtualMemory()
if err != nil {
return nil, err
}
return []*pageFileStats{{
deviceName: "", // We do not support per-device swap
usedBytes: vmem.SwapTotal - vmem.SwapFree - vmem.SwapCached,
freeBytes: vmem.SwapFree,
cachedBytes: &vmem.SwapCached,
}}, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package pagingscraper

import (
"sync"
"syscall"
"unsafe"

Expand All @@ -32,6 +33,11 @@ var (
procEnumPageFilesW = modPsapi.NewProc("EnumPageFilesW")
)

var (
pageSize uint64
pageSizeOnce sync.Once
)

type systemInfo struct {
wProcessorArchitecture uint16
wReserved uint16
Expand All @@ -52,12 +58,6 @@ func getPageSize() uint64 {
return uint64(sysInfo.dwPageSize)
}

type pageFileData struct {
name string
usedPages uint64
totalPages uint64
}

// system type as defined in https://docs.microsoft.com/en-us/windows/win32/api/psapi/ns-psapi-enum_page_file_information
type enumPageFileInformation struct {
cb uint32
Expand All @@ -67,10 +67,12 @@ type enumPageFileInformation struct {
peakUsage uint64
}

func getPageFileStats() ([]*pageFileData, error) {
func getPageFileStats() ([]*pageFileStats, error) {
pageSizeOnce.Do(func() { pageSize = getPageSize() })

// the following system call invokes the supplied callback function once for each page file before returning
// see https://docs.microsoft.com/en-us/windows/win32/api/psapi/nf-psapi-enumpagefilesw
var pageFiles []*pageFileData
var pageFiles []*pageFileStats
result, _, _ := procEnumPageFilesW.Call(windows.NewCallback(pEnumPageFileCallbackW), uintptr(unsafe.Pointer(&pageFiles)))
if result == 0 {
return nil, windows.GetLastError()
Expand All @@ -80,13 +82,13 @@ func getPageFileStats() ([]*pageFileData, error) {
}

// system callback as defined in https://docs.microsoft.com/en-us/windows/win32/api/psapi/nc-psapi-penum_page_file_callbackw
func pEnumPageFileCallbackW(pageFiles *[]*pageFileData, enumPageFileInfo *enumPageFileInformation, lpFilenamePtr *[syscall.MAX_LONG_PATH]uint16) *bool {
func pEnumPageFileCallbackW(pageFiles *[]*pageFileStats, enumPageFileInfo *enumPageFileInformation, lpFilenamePtr *[syscall.MAX_LONG_PATH]uint16) *bool {
pageFileName := syscall.UTF16ToString((*lpFilenamePtr)[:])

pfData := &pageFileData{
name: pageFileName,
usedPages: enumPageFileInfo.totalInUse,
totalPages: enumPageFileInfo.totalSize,
pfData := &pageFileStats{
deviceName: pageFileName,
usedBytes: enumPageFileInfo.totalInUse * pageSize,
freeBytes: (enumPageFileInfo.totalSize - enumPageFileInfo.totalInUse) * pageSize,
}

*pageFiles = append(*pageFiles, pfData)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@ type scraper struct {
startTime pdata.Timestamp

// for mocking
bootTime func() (uint64, error)
virtualMemory func() (*mem.VirtualMemoryStat, error)
swapMemory func() (*mem.SwapMemoryStat, error)
bootTime func() (uint64, error)
getPageFileStats func() ([]*pageFileStats, error)
swapMemory func() (*mem.SwapMemoryStat, error)
}

// newPagingScraper creates a Paging Scraper
func newPagingScraper(_ context.Context, cfg *Config) *scraper {
return &scraper{config: cfg, bootTime: host.BootTime, virtualMemory: mem.VirtualMemory, swapMemory: mem.SwapMemory}
return &scraper{config: cfg, bootTime: host.BootTime, getPageFileStats: getPageFileStats, swapMemory: mem.SwapMemory}
}

func (s *scraper) start(context.Context, component.Host) error {
Expand Down Expand Up @@ -81,28 +81,35 @@ func (s *scraper) scrape(_ context.Context) (pdata.MetricSlice, error) {

func (s *scraper) scrapeAndAppendPagingUsageMetric(metrics pdata.MetricSlice) error {
now := pdata.NewTimestampFromTime(time.Now())
vmem, err := s.virtualMemory()
pageFileStats, err := s.getPageFileStats()
if err != nil {
return err
}

idx := metrics.Len()
metrics.EnsureCapacity(idx + pagingUsageMetricsLen)
initializePagingUsageMetric(metrics.AppendEmpty(), now, vmem)
initializePagingUsageMetric(metrics.AppendEmpty(), now, pageFileStats)
return nil
}

func initializePagingUsageMetric(metric pdata.Metric, now pdata.Timestamp, vmem *mem.VirtualMemoryStat) {
func initializePagingUsageMetric(metric pdata.Metric, now pdata.Timestamp, pageFileStats []*pageFileStats) {
metadata.Metrics.SystemPagingUsage.Init(metric)

idps := metric.Sum().DataPoints()
idps.EnsureCapacity(3)
initializePagingUsageDataPoint(idps.AppendEmpty(), now, metadata.LabelState.Used, int64(vmem.SwapTotal-vmem.SwapFree-vmem.SwapCached))
initializePagingUsageDataPoint(idps.AppendEmpty(), now, metadata.LabelState.Free, int64(vmem.SwapFree))
initializePagingUsageDataPoint(idps.AppendEmpty(), now, metadata.LabelState.Cached, int64(vmem.SwapCached))
for _, pageFile := range pageFileStats {
initializePagingUsageDataPoint(idps.AppendEmpty(), now, pageFile.deviceName, metadata.LabelState.Used, int64(pageFile.usedBytes))
initializePagingUsageDataPoint(idps.AppendEmpty(), now, pageFile.deviceName, metadata.LabelState.Free, int64(pageFile.freeBytes))
if pageFile.cachedBytes != nil {
initializePagingUsageDataPoint(idps.AppendEmpty(), now, pageFile.deviceName, metadata.LabelState.Cached, int64(*pageFile.cachedBytes))
}
}
}

func initializePagingUsageDataPoint(dataPoint pdata.NumberDataPoint, now pdata.Timestamp, stateLabel string, value int64) {
func initializePagingUsageDataPoint(dataPoint pdata.NumberDataPoint, now pdata.Timestamp, deviceLabel, stateLabel string, value int64) {
if deviceLabel != "" {
dataPoint.Attributes().InsertString(metadata.Labels.Device, deviceLabel)
}
dataPoint.Attributes().InsertString(metadata.Labels.State, stateLabel)
dataPoint.SetTimestamp(now)
dataPoint.SetIntVal(value)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
func TestScrape_Errors(t *testing.T) {
type testCase struct {
name string
virtualMemoryFunc func() (*mem.VirtualMemoryStat, error)
virtualMemoryFunc func() ([]*pageFileStats, error)
swapMemoryFunc func() (*mem.SwapMemoryStat, error)
expectedError string
expectedErrCount int
Expand All @@ -41,7 +41,7 @@ func TestScrape_Errors(t *testing.T) {
testCases := []testCase{
{
name: "virtualMemoryError",
virtualMemoryFunc: func() (*mem.VirtualMemoryStat, error) { return nil, errors.New("err1") },
virtualMemoryFunc: func() ([]*pageFileStats, error) { return nil, errors.New("err1") },
expectedError: "err1",
expectedErrCount: pagingUsageMetricsLen,
},
Expand All @@ -53,7 +53,7 @@ func TestScrape_Errors(t *testing.T) {
},
{
name: "multipleErrors",
virtualMemoryFunc: func() (*mem.VirtualMemoryStat, error) { return nil, errors.New("err1") },
virtualMemoryFunc: func() ([]*pageFileStats, error) { return nil, errors.New("err1") },
swapMemoryFunc: func() (*mem.SwapMemoryStat, error) { return nil, errors.New("err2") },
expectedError: "err1; err2",
expectedErrCount: pagingUsageMetricsLen + pagingMetricsLen,
Expand All @@ -64,7 +64,7 @@ func TestScrape_Errors(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
scraper := newPagingScraper(context.Background(), &Config{})
if test.virtualMemoryFunc != nil {
scraper.virtualMemory = test.virtualMemoryFunc
scraper.getPageFileStats = test.virtualMemoryFunc
}
if test.swapMemoryFunc != nil {
scraper.swapMemory = test.swapMemoryFunc
Expand Down
Loading