From e412c00c49e39620dabf5d730c862ec4d7fe3b75 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Jul 2016 02:29:27 -0400 Subject: [PATCH] Add freebsd build tags for Metricbeat system metricsets (#1966) The MetricSets made available on FreeBSD by this change are core, cpu, filesystem, fsstat, memory, and process. - Enable python system tests on FreeBSD. - Remove cmdline from the required fields list in the python system tests for system/process events because cmdline may not be included for certain kernel level processes. - Disable python filtering tests on OpenBSD because system/process is not implemented. - Add build tags for the Golang tests in the system module. --- metricbeat/Makefile | 2 +- metricbeat/docs/faq.asciidoc | 12 +++++++ .../module/system/common/process_test.go | 1 + .../module/system/core/_meta/docs.asciidoc | 7 ++++ metricbeat/module/system/core/core.go | 2 +- metricbeat/module/system/core/core_test.go | 3 ++ .../module/system/cpu/_meta/docs.asciidoc | 8 +++++ metricbeat/module/system/cpu/cpu.go | 2 +- metricbeat/module/system/cpu/cpu_test.go | 3 ++ .../system/filesystem/_meta/docs.asciidoc | 11 ++++++- .../module/system/filesystem/filesystem.go | 2 +- .../system/filesystem/filesystem_test.go | 3 ++ .../module/system/fsstat/_meta/docs.asciidoc | 8 +++++ metricbeat/module/system/fsstat/fsstat.go | 2 +- .../module/system/fsstat/fsstat_test.go | 3 ++ .../module/system/memory/_meta/docs.asciidoc | 8 +++++ metricbeat/module/system/memory/memory.go | 2 +- .../module/system/memory/memory_test.go | 3 ++ .../module/system/network/network_test.go | 3 ++ .../module/system/process/_meta/docs.asciidoc | 10 +++++- metricbeat/module/system/process/process.go | 2 +- .../module/system/process/process_test.go | 5 ++- metricbeat/tests/system/test_base.py | 2 +- metricbeat/tests/system/test_filtering.py | 2 +- metricbeat/tests/system/test_system.py | 32 ++++++++++++------- 25 files changed, 114 insertions(+), 24 deletions(-) diff --git a/metricbeat/Makefile b/metricbeat/Makefile index 78035be95ad7..6ad1d68a210e 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -10,7 +10,7 @@ ES_BEATS?=.. # Metricbeat can only be cross-compiled on platforms not requiring CGO which # are the same platforms where the system metrics (cpu, memory) are not # implemented. -GOX_OS=solaris freebsd netbsd +GOX_OS=solaris netbsd GOX_FLAGS='-arch=amd64' include ${ES_BEATS}/libbeat/scripts/Makefile diff --git a/metricbeat/docs/faq.asciidoc b/metricbeat/docs/faq.asciidoc index 24dffeb6c997..4f3279eb7464 100644 --- a/metricbeat/docs/faq.asciidoc +++ b/metricbeat/docs/faq.asciidoc @@ -4,4 +4,16 @@ This section contains frequently asked questions about Metricbeat. Also check out the https://discuss.elastic.co/c/beats/metricbeat[Metricbeat discussion forum]. +=== How do I fix the `open /compat/linux/proc: no such file or directory` error on FreeBSD? + +The system metricsets rely a Linux comparability layer to retrieve metrics on +FreeBSD. You need to mount the Linux procfs filesystem using the following +commands. + +[source,sh] +---- +sudo mkdir -p /compat/linux/proc +sudo mount -t linprocfs /dev/null /compat/linux/proc +---- + include::../../libbeat/docs/shared-faq.asciidoc[] diff --git a/metricbeat/module/system/common/process_test.go b/metricbeat/module/system/common/process_test.go index 00c4c9fa0f2a..0479f4eb8957 100644 --- a/metricbeat/module/system/common/process_test.go +++ b/metricbeat/module/system/common/process_test.go @@ -1,4 +1,5 @@ // +build !integration +// +build darwin freebsd linux windows package common diff --git a/metricbeat/module/system/core/_meta/docs.asciidoc b/metricbeat/module/system/core/_meta/docs.asciidoc index 9354669aaac5..90b5106546cc 100644 --- a/metricbeat/module/system/core/_meta/docs.asciidoc +++ b/metricbeat/module/system/core/_meta/docs.asciidoc @@ -1,3 +1,10 @@ === System Core Metricset The System `core` metricset provides load statistics for each CPU core. + +This metricset is available on: + +- Darwin +- FreeBSD +- Linux +- OpenBSD diff --git a/metricbeat/module/system/core/core.go b/metricbeat/module/system/core/core.go index 7f6db794b377..7acb87c5cfde 100644 --- a/metricbeat/module/system/core/core.go +++ b/metricbeat/module/system/core/core.go @@ -1,4 +1,4 @@ -// +build darwin linux openbsd windows +// +build darwin freebsd linux openbsd package core diff --git a/metricbeat/module/system/core/core_test.go b/metricbeat/module/system/core/core_test.go index 00114298ca84..e6e87b774bb1 100644 --- a/metricbeat/module/system/core/core_test.go +++ b/metricbeat/module/system/core/core_test.go @@ -1,3 +1,6 @@ +// +build !integration +// +build darwin freebsd linux openbsd + package core import ( diff --git a/metricbeat/module/system/cpu/_meta/docs.asciidoc b/metricbeat/module/system/cpu/_meta/docs.asciidoc index 18c6a98f35f9..1a62e46bd01e 100644 --- a/metricbeat/module/system/cpu/_meta/docs.asciidoc +++ b/metricbeat/module/system/cpu/_meta/docs.asciidoc @@ -1,3 +1,11 @@ === System CPU Metricset The System `cpu` metricset provides CPU statistics. + +This metricset is available on: + +- Darwin +- FreeBSD +- Linux +- OpenBSD +- Windows diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index 5af23044082c..89a189538577 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -1,4 +1,4 @@ -// +build darwin linux openbsd windows +// +build darwin freebsd linux openbsd windows package cpu diff --git a/metricbeat/module/system/cpu/cpu_test.go b/metricbeat/module/system/cpu/cpu_test.go index a93082ade0f1..a6e81f81606c 100644 --- a/metricbeat/module/system/cpu/cpu_test.go +++ b/metricbeat/module/system/cpu/cpu_test.go @@ -1,3 +1,6 @@ +// +build !integration +// +build darwin freebsd linux openbsd windows + package cpu import ( diff --git a/metricbeat/module/system/filesystem/_meta/docs.asciidoc b/metricbeat/module/system/filesystem/_meta/docs.asciidoc index b9f90aaa30ef..de21aa3d6826 100644 --- a/metricbeat/module/system/filesystem/_meta/docs.asciidoc +++ b/metricbeat/module/system/filesystem/_meta/docs.asciidoc @@ -1,3 +1,12 @@ === System Filesystem Metricset -The System `filesystem` metricset provides file system statistics. For each file system, one document is provided. +The System `filesystem` metricset provides file system statistics. For each file +system, one document is provided. + +This metricset is available on: + +- Darwin +- FreeBSD +- Linux +- OpenBSD +- Windows diff --git a/metricbeat/module/system/filesystem/filesystem.go b/metricbeat/module/system/filesystem/filesystem.go index 3ee624e58093..425b5572209f 100644 --- a/metricbeat/module/system/filesystem/filesystem.go +++ b/metricbeat/module/system/filesystem/filesystem.go @@ -1,4 +1,4 @@ -// +build darwin linux openbsd windows +// +build darwin freebsd linux openbsd windows package filesystem diff --git a/metricbeat/module/system/filesystem/filesystem_test.go b/metricbeat/module/system/filesystem/filesystem_test.go index 583123ba144e..334d3be18df0 100644 --- a/metricbeat/module/system/filesystem/filesystem_test.go +++ b/metricbeat/module/system/filesystem/filesystem_test.go @@ -1,3 +1,6 @@ +// +build !integration +// +build darwin freebsd linux openbsd windows + package filesystem import ( diff --git a/metricbeat/module/system/fsstat/_meta/docs.asciidoc b/metricbeat/module/system/fsstat/_meta/docs.asciidoc index 33fcef8a57eb..4eecef32e969 100644 --- a/metricbeat/module/system/fsstat/_meta/docs.asciidoc +++ b/metricbeat/module/system/fsstat/_meta/docs.asciidoc @@ -1,3 +1,11 @@ === System Fsstat Metricset The System `fsstats` metricset provides overall file system statistics. + +This metricset is available on: + +- Darwin +- FreeBSD +- Linux +- OpenBSD +- Windows diff --git a/metricbeat/module/system/fsstat/fsstat.go b/metricbeat/module/system/fsstat/fsstat.go index 884574f45811..16a69a4c2987 100644 --- a/metricbeat/module/system/fsstat/fsstat.go +++ b/metricbeat/module/system/fsstat/fsstat.go @@ -1,4 +1,4 @@ -// +build darwin linux openbsd windows +// +build darwin freebsd linux openbsd windows package fsstat diff --git a/metricbeat/module/system/fsstat/fsstat_test.go b/metricbeat/module/system/fsstat/fsstat_test.go index 7f924b9f6f2f..4c3a0a1628b9 100644 --- a/metricbeat/module/system/fsstat/fsstat_test.go +++ b/metricbeat/module/system/fsstat/fsstat_test.go @@ -1,3 +1,6 @@ +// +build !integration +// +build darwin freebsd linux openbsd windows + package fsstat import ( diff --git a/metricbeat/module/system/memory/_meta/docs.asciidoc b/metricbeat/module/system/memory/_meta/docs.asciidoc index 4f3dff1396b7..c116a45444df 100644 --- a/metricbeat/module/system/memory/_meta/docs.asciidoc +++ b/metricbeat/module/system/memory/_meta/docs.asciidoc @@ -1,3 +1,11 @@ === System Memory Metricset The System `memory` metricset provides memory statistics. + +This metricset is available on: + +- Darwin +- FreeBSD +- Linux +- OpenBSD +- Windows diff --git a/metricbeat/module/system/memory/memory.go b/metricbeat/module/system/memory/memory.go index 811e45b4e9a7..10c3f582ed65 100644 --- a/metricbeat/module/system/memory/memory.go +++ b/metricbeat/module/system/memory/memory.go @@ -1,4 +1,4 @@ -// +build darwin linux openbsd windows +// +build darwin freebsd linux openbsd windows package memory diff --git a/metricbeat/module/system/memory/memory_test.go b/metricbeat/module/system/memory/memory_test.go index bf8e5c831d56..e680344ceb57 100644 --- a/metricbeat/module/system/memory/memory_test.go +++ b/metricbeat/module/system/memory/memory_test.go @@ -1,3 +1,6 @@ +// +build !integration +// +build darwin freebsd linux openbsd windows + package memory import ( diff --git a/metricbeat/module/system/network/network_test.go b/metricbeat/module/system/network/network_test.go index 3f81f56c1244..04c63f60634f 100644 --- a/metricbeat/module/system/network/network_test.go +++ b/metricbeat/module/system/network/network_test.go @@ -1,3 +1,6 @@ +// +build !integration +// +build darwin freebsd linux windows + package network import ( diff --git a/metricbeat/module/system/process/_meta/docs.asciidoc b/metricbeat/module/system/process/_meta/docs.asciidoc index 239239a809b2..2a8412c06b73 100644 --- a/metricbeat/module/system/process/_meta/docs.asciidoc +++ b/metricbeat/module/system/process/_meta/docs.asciidoc @@ -1,3 +1,11 @@ === System Process Metricset -The System `process` metricset provides process statistics. One document is provided for each process. +The System `process` metricset provides process statistics. One document is +provided for each process. + +This metricset is available on: + +- Darwin +- FreeBSD +- Linux +- Windows diff --git a/metricbeat/module/system/process/process.go b/metricbeat/module/system/process/process.go index 49f120903cf4..9ce13892f907 100644 --- a/metricbeat/module/system/process/process.go +++ b/metricbeat/module/system/process/process.go @@ -1,4 +1,4 @@ -// +build darwin linux windows +// +build darwin freebsd linux windows package process diff --git a/metricbeat/module/system/process/process_test.go b/metricbeat/module/system/process/process_test.go index c4471284a99d..494a0a795a88 100644 --- a/metricbeat/module/system/process/process_test.go +++ b/metricbeat/module/system/process/process_test.go @@ -1,3 +1,6 @@ +// +build !integration +// +build darwin freebsd linux windows + package process import ( @@ -11,7 +14,7 @@ import ( func TestData(t *testing.T) { f := mbtest.NewEventsFetcher(t, getConfig()) - // Do a first fetch to have precentages + // Do a first fetch to have percentages f.Fetch() time.Sleep(1 * time.Second) diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index 58fc3a8fe6f2..664f90818f2b 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -5,7 +5,7 @@ class Test(BaseTest): - @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_start_stop(self): """ Metricbeat starts and stops without error. diff --git a/metricbeat/tests/system/test_filtering.py b/metricbeat/tests/system/test_filtering.py index ab60c91215c5..31b9b746e5f9 100644 --- a/metricbeat/tests/system/test_filtering.py +++ b/metricbeat/tests/system/test_filtering.py @@ -3,7 +3,7 @@ import metricbeat import unittest -@unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os") +@unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") class GlobalFiltering(metricbeat.BaseTest): def test_drop_fields(self): diff --git a/metricbeat/tests/system/test_system.py b/metricbeat/tests/system/test_system.py index 3d3750490861..c5f4cb7b5144 100644 --- a/metricbeat/tests/system/test_system.py +++ b/metricbeat/tests/system/test_system.py @@ -31,12 +31,14 @@ SYSTEM_NETWORK_FIELDS = ["name", "out.bytes", "in.bytes", "out.packets", "in.packets", "in.error", "out.error", "in.dropeed", "out.dropped"] -SYSTEM_PROCESS_FIELDS = ["cmdline", "cpu", "memory", "name", "pid", "ppid", - "state", "username"] +# cmdline is also part of the system process fields, but it may not be present +# for some kernel level processes. +SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "name", "pid", "ppid", "state", + "username"] class SystemTest(metricbeat.BaseTest): - @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_cpu(self): """ Test cpu system output. @@ -62,7 +64,7 @@ def test_cpu(self): cpu = evt["system"]["cpu"] self.assertItemsEqual(self.de_dot(SYSTEM_CPU_FIELDS), cpu.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_cpu_ticks_option(self): """ Test cpu_ticks configuration option. @@ -91,7 +93,7 @@ def test_cpu_ticks_option(self): cpuStats = evt["system"]["cpu"] self.assertItemsEqual(self.de_dot(SYSTEM_CPU_FIELDS_ALL), cpuStats.keys()) - @unittest.skipUnless(re.match("(?i)linux|darwin|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") def test_core(self): """ Test core system output. @@ -117,8 +119,8 @@ def test_core(self): core = evt["system"]["core"] self.assertItemsEqual(self.de_dot(SYSTEM_CORE_FIELDS), core.keys()) - @unittest.skipUnless(re.match("(?i)linux|darwin|openbsd", sys.platform), "os") - def test_core(self): + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") + def test_core_with_cpu_ticks(self): """ Test core system output. """ @@ -172,7 +174,7 @@ def test_diskio(self): diskio = evt["system"]["diskio"] self.assertItemsEqual(self.de_dot(SYSTEM_DISKIO_FIELDS), diskio.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_filesystem(self): """ Test system/filesystem output. @@ -198,7 +200,7 @@ def test_filesystem(self): filesystem = evt["system"]["filesystem"] self.assertItemsEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS), filesystem.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_fsstat(self): """ Test system/fsstat output. @@ -224,7 +226,7 @@ def test_fsstat(self): fsstat = evt["system"]["fsstat"] self.assertItemsEqual(SYSTEM_FSSTAT_FIELDS, fsstat.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_memory(self): """ Test system memory output. @@ -290,7 +292,7 @@ def test_network(self): network = evt["system"]["network"] self.assertItemsEqual(self.de_dot(SYSTEM_NETWORK_FIELDS), network.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process(self): """ Test system/process output. @@ -311,11 +313,18 @@ def test_process(self): output = self.read_output_json() self.assertGreater(len(output), 0) + found_cmdline = False for evt in output: self.assert_fields_are_documented(evt) process = evt["system"]["process"] + cmdline = process.pop("cmdline", None) + if cmdline is not None: + found_cmdline = True self.assertItemsEqual(SYSTEM_PROCESS_FIELDS, process.keys()) + self.assertTrue(found_cmdline, "cmdline not found in any process events") + + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process_metricbeat(self): """ Checks that the per proc stats are found in the output and @@ -340,7 +349,6 @@ def test_process_metricbeat(self): assert isinstance(output["system.process.cpu.start_time"], basestring) self.check_username(output["system.process.username"]) - def check_username(self, observed, expected = None): if expected == None: expected = getpass.getuser()