Add freebsd build tags for Metricbeat system metricsets (elastic#1966)

The MetricSets made available on FreeBSD by this change are core, cpu, filesystem, fsstat, memory, and process.

- Enable python system tests on FreeBSD.
- Remove cmdline from the required fields list in the python system tests for system/process events because cmdline may not be included for certain kernel level processes.
- Disable python filtering tests on OpenBSD because system/process is not implemented.
- Add build tags for the Golang tests in the system module.

metricbeat/Makefile

@@ -10,7 +10,7 @@ ES_BEATS?=..
 # Metricbeat can only be cross-compiled on platforms not requiring CGO which
 # are the same platforms where the system metrics (cpu, memory) are not
 # implemented.
-GOX_OS=solaris freebsd netbsd
+GOX_OS=solaris netbsd
 GOX_FLAGS='-arch=amd64'
 
 include ${ES_BEATS}/libbeat/scripts/Makefile

metricbeat/docs/faq.asciidoc

@@ -4,4 +4,16 @@
 This section contains frequently asked questions about Metricbeat. Also check out the
 https://discuss.elastic.co/c/beats/metricbeat[Metricbeat discussion forum].
 
+=== How do I fix the `open /compat/linux/proc: no such file or directory` error on FreeBSD?
+
+The system metricsets rely a Linux comparability layer to retrieve metrics on
+FreeBSD. You need to mount the Linux procfs filesystem using the following
+commands.
+
+[source,sh]
+----
+sudo mkdir -p /compat/linux/proc
+sudo mount -t linprocfs /dev/null /compat/linux/proc
+----
+
 include::../../libbeat/docs/shared-faq.asciidoc[]

metricbeat/module/system/common/process_test.go

@@ -1,4 +1,5 @@
 // +build !integration
+// +build darwin freebsd linux windows
 
 package common
 

metricbeat/module/system/core/_meta/docs.asciidoc

@@ -1,3 +1,10 @@
 === System Core Metricset
 
 The System `core` metricset provides load statistics for each CPU core.
+
+This metricset is available on:
+
+- Darwin
+- FreeBSD
+- Linux
+- OpenBSD

metricbeat/module/system/core/core.go

@@ -1,4 +1,4 @@
-// +build darwin linux openbsd windows
+// +build darwin freebsd linux openbsd
 
 package core
 

metricbeat/module/system/core/core_test.go

@@ -1,3 +1,6 @@
+// +build !integration
+// +build darwin freebsd linux openbsd
+
 package core
 
 import (

metricbeat/module/system/cpu/_meta/docs.asciidoc

@@ -1,3 +1,11 @@
 === System CPU Metricset
 
 The System `cpu` metricset provides CPU statistics.
+
+This metricset is available on:
+
+- Darwin
+- FreeBSD
+- Linux
+- OpenBSD
+- Windows

metricbeat/module/system/cpu/cpu.go

@@ -1,4 +1,4 @@
-// +build darwin linux openbsd windows
+// +build darwin freebsd linux openbsd windows
 
 package cpu
 

metricbeat/module/system/cpu/cpu_test.go

@@ -1,3 +1,6 @@
+// +build !integration
+// +build darwin freebsd linux openbsd windows
+
 package cpu
 
 import (

metricbeat/module/system/filesystem/_meta/docs.asciidoc

@@ -1,3 +1,12 @@
 === System Filesystem Metricset
 
-The System `filesystem` metricset provides file system statistics. For each file system, one document is provided.
+The System `filesystem` metricset provides file system statistics. For each file
+system, one document is provided.
+
+This metricset is available on:
+
+- Darwin
+- FreeBSD
+- Linux
+- OpenBSD
+- Windows

metricbeat/module/system/filesystem/filesystem.go

@@ -1,4 +1,4 @@
-// +build darwin linux openbsd windows
+// +build darwin freebsd linux openbsd windows
 
 package filesystem
 

metricbeat/module/system/filesystem/filesystem_test.go

@@ -1,3 +1,6 @@
+// +build !integration
+// +build darwin freebsd linux openbsd windows
+
 package filesystem
 
 import (

metricbeat/module/system/fsstat/_meta/docs.asciidoc

@@ -1,3 +1,11 @@
 === System Fsstat Metricset
 
 The System `fsstats` metricset provides overall file system statistics.
+
+This metricset is available on:
+
+- Darwin
+- FreeBSD
+- Linux
+- OpenBSD
+- Windows

metricbeat/module/system/fsstat/fsstat.go

@@ -1,4 +1,4 @@
-// +build darwin linux openbsd windows
+// +build darwin freebsd linux openbsd windows
 
 package fsstat
 

metricbeat/module/system/fsstat/fsstat_test.go

@@ -1,3 +1,6 @@
+// +build !integration
+// +build darwin freebsd linux openbsd windows
+
 package fsstat
 
 import (

metricbeat/module/system/memory/_meta/docs.asciidoc

@@ -1,3 +1,11 @@
 === System Memory Metricset
 
 The System `memory` metricset provides memory statistics.
+
+This metricset is available on:
+
+- Darwin
+- FreeBSD
+- Linux
+- OpenBSD
+- Windows

metricbeat/module/system/memory/memory.go

@@ -1,4 +1,4 @@
-// +build darwin linux openbsd windows
+// +build darwin freebsd linux openbsd windows
 
 package memory
 

metricbeat/module/system/memory/memory_test.go

@@ -1,3 +1,6 @@
+// +build !integration
+// +build darwin freebsd linux openbsd windows
+
 package memory
 
 import (

metricbeat/module/system/network/network_test.go

@@ -1,3 +1,6 @@
+// +build !integration
+// +build darwin freebsd linux windows
+
 package network
 
 import (

metricbeat/module/system/process/_meta/docs.asciidoc

@@ -1,3 +1,11 @@
 === System Process Metricset
 
-The System `process` metricset provides process statistics. One document is provided for each process.
+The System `process` metricset provides process statistics. One document is
+provided for each process.
+
+This metricset is available on:
+
+- Darwin
+- FreeBSD
+- Linux
+- Windows

metricbeat/module/system/process/process.go

@@ -1,4 +1,4 @@
-// +build darwin linux windows
+// +build darwin freebsd linux windows
 
 package process
 

metricbeat/module/system/process/process_test.go

@@ -1,3 +1,6 @@
+// +build !integration
+// +build darwin freebsd linux windows
+
 package process
 
 import (
@@ -11,7 +14,7 @@ import (
 func TestData(t *testing.T) {
 	f := mbtest.NewEventsFetcher(t, getConfig())
 
-	// Do a first fetch to have precentages
+	// Do a first fetch to have percentages
 	f.Fetch()
 	time.Sleep(1 * time.Second)
 

metricbeat/tests/system/test_base.py

@@ -5,7 +5,7 @@
 
 
 class Test(BaseTest):
-    @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_start_stop(self):
         """
         Metricbeat starts and stops without error.

metricbeat/tests/system/test_filtering.py

@@ -3,7 +3,7 @@
 import metricbeat
 import unittest
 
-@unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os")
+@unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os")
 class GlobalFiltering(metricbeat.BaseTest):
 
     def test_drop_fields(self):

metricbeat/tests/system/test_system.py

@@ -31,12 +31,14 @@
 SYSTEM_NETWORK_FIELDS = ["name", "out.bytes", "in.bytes", "out.packets",
                          "in.packets", "in.error", "out.error", "in.dropeed", "out.dropped"]
 
-SYSTEM_PROCESS_FIELDS = ["cmdline", "cpu", "memory", "name", "pid", "ppid",
-                         "state", "username"]
+# cmdline is also part of the system process fields, but it may not be present
+# for some kernel level processes.
+SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "name", "pid", "ppid", "state",
+                         "username"]
 
 
 class SystemTest(metricbeat.BaseTest):
-    @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_cpu(self):
         """
         Test cpu system output.
@@ -62,7 +64,7 @@ def test_cpu(self):
         cpu = evt["system"]["cpu"]
         self.assertItemsEqual(self.de_dot(SYSTEM_CPU_FIELDS), cpu.keys())
 
-    @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_cpu_ticks_option(self):
         """
         Test cpu_ticks configuration option.
@@ -91,7 +93,7 @@ def test_cpu_ticks_option(self):
             cpuStats = evt["system"]["cpu"]
             self.assertItemsEqual(self.de_dot(SYSTEM_CPU_FIELDS_ALL), cpuStats.keys())
 
-    @unittest.skipUnless(re.match("(?i)linux|darwin|openbsd", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_core(self):
         """
         Test core system output.
@@ -117,8 +119,8 @@ def test_core(self):
             core = evt["system"]["core"]
             self.assertItemsEqual(self.de_dot(SYSTEM_CORE_FIELDS), core.keys())
 
-    @unittest.skipUnless(re.match("(?i)linux|darwin|openbsd", sys.platform), "os")
-    def test_core(self):
+    @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os")
+    def test_core_with_cpu_ticks(self):
         """
         Test core system output.
         """
@@ -172,7 +174,7 @@ def test_diskio(self):
             diskio = evt["system"]["diskio"]
             self.assertItemsEqual(self.de_dot(SYSTEM_DISKIO_FIELDS), diskio.keys())
 
-    @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_filesystem(self):
         """
         Test system/filesystem output.
@@ -198,7 +200,7 @@ def test_filesystem(self):
             filesystem = evt["system"]["filesystem"]
             self.assertItemsEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS), filesystem.keys())
 
-    @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_fsstat(self):
         """
         Test system/fsstat output.
@@ -224,7 +226,7 @@ def test_fsstat(self):
         fsstat = evt["system"]["fsstat"]
         self.assertItemsEqual(SYSTEM_FSSTAT_FIELDS, fsstat.keys())
 
-    @unittest.skipUnless(re.match("(?i)win|linux|darwin|openbsd", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_memory(self):
         """
         Test system memory output.
@@ -290,7 +292,7 @@ def test_network(self):
             network = evt["system"]["network"]
             self.assertItemsEqual(self.de_dot(SYSTEM_NETWORK_FIELDS), network.keys())
 
-    @unittest.skipUnless(re.match("(?i)win|linux|darwin", sys.platform), "os")
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os")
     def test_process(self):
         """
         Test system/process output.
@@ -311,11 +313,18 @@ def test_process(self):
         output = self.read_output_json()
         self.assertGreater(len(output), 0)
 
+        found_cmdline = False
         for evt in output:
             self.assert_fields_are_documented(evt)
             process = evt["system"]["process"]
+            cmdline = process.pop("cmdline", None)
+            if cmdline is not None:
+                found_cmdline = True
             self.assertItemsEqual(SYSTEM_PROCESS_FIELDS, process.keys())
 
+        self.assertTrue(found_cmdline, "cmdline not found in any process events")
+
+    @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os")
     def test_process_metricbeat(self):
         """
         Checks that the per proc stats are found in the output and
@@ -340,7 +349,6 @@ def test_process_metricbeat(self):
         assert isinstance(output["system.process.cpu.start_time"], basestring)
         self.check_username(output["system.process.username"])
 
-
     def check_username(self, observed, expected = None):
         if expected == None:
             expected = getpass.getuser()