metricbeat: collect total thread count per process (elastic#36578)

* metricbeat: collect number of threads per process

Upgrades elastic-agent-system-metrics to v0.7.0 in order to collect the number of threads per process.

CHANGELOG.next.asciidoc

@@ -24,6 +24,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Metricbeat*
 
+ - System module now collects the number of threads per process.
+The elastic-agent-system-metrics was updated to v0.7.0 as the number of threads
+is collected by it.
 
 *Osquerybeat*
 

NOTICE.txt

@@ -13026,11 +13026,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.6.1
+Version: v0.7.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.6.1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.7.0/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -34256,11 +34256,11 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/cyphar/filepath-securejoin
-Version: v0.2.3
+Version: v0.2.4
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/cyphar/filepath-securejoin@v0.2.3/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/cyphar/filepath-securejoin@v0.2.4/LICENSE:
 
 Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved.
 Copyright (C) 2017 SUSE LLC. All rights reserved.

go.mod

@@ -204,7 +204,7 @@ require (
 	github.com/elastic/elastic-agent-autodiscover v0.6.2
 	github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
-	github.com/elastic/elastic-agent-system-metrics v0.6.1
+	github.com/elastic/elastic-agent-system-metrics v0.7.0
 	github.com/elastic/go-elasticsearch/v8 v8.10.0
 	github.com/elastic/mito v1.6.0
 	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
@@ -261,7 +261,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/dgraph-io/ristretto v0.1.0 // indirect

go.sum

@@ -550,8 +550,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
-github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
-github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -657,8 +657,8 @@ github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a
 github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
-github.com/elastic/elastic-agent-system-metrics v0.6.1 h1:LCN1lvQTkdUuU/rKlpKyVMDU/G/I8/iZWCaW6K+mo4o=
-github.com/elastic/elastic-agent-system-metrics v0.6.1/go.mod h1:Bj8XM/uNKm553blQHkGNEICRLGnVEtw8yttmV5vBngA=
+github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ=
+github.com/elastic/elastic-agent-system-metrics v0.7.0/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
 github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=
 github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=

libbeat/generator/fields/module_fields_collector.go

@@ -19,6 +19,7 @@ package fields
 
 import (
 	"io/ioutil"
+	"os"
 	"path/filepath"
 )
 
@@ -30,9 +31,9 @@ var indentByModule = map[string]int{
 	"protos":     8,
 }
 
-// GetModules returns a the list of modules for the given modules directory
+// GetModules returns a list of modules for the given modules directory
 func GetModules(modulesDir string) ([]string, error) {
-	moduleInfos, err := ioutil.ReadDir(modulesDir)
+	moduleInfos, err := os.ReadDir(modulesDir)
 	if err != nil {
 		return nil, err
 	}

libbeat/tests/system/beat/beat.py

@@ -717,6 +717,7 @@ def extract_fields(doc_list, name):
             if not os.path.isfile(path):
                 path = os.path.abspath(os.path.dirname(
                     __file__) + "../../../../_meta/fields.common.yml")
+
             with open(path, encoding="utf-8") as f2:
                 content = f2.read()
 
@@ -788,7 +789,7 @@ def output_count(self, pred, output_file=None):
     def assert_fields_are_documented(self, evt):
         """
         Assert that all keys present in evt are documented in fields.yml.
-        This reads from the global fields.yml, means `make collect` has to be run before the check.
+        This reads from the global fields.yml, means `mage fields` has to be run before the check.
         """
         expected_fields, dict_fields, aliases = self.load_fields()
         flat = self.flatten_object(evt, dict_fields)

metricbeat/docs/fields.asciidoc

@@ -62056,6 +62056,16 @@ alias to: process.pgid
 
 --
 
+*`system.process.num_threads`*::
++
+--
+Number of threads in the process
+
+
+type: integer
+
+--
+
 *`system.process.cmdline`*::
 +
 --

metricbeat/module/system/process/_meta/data.json

@@ -250,10 +250,11 @@
                 "share": 14946304,
                 "size": 1725992960
             },
-            "state": "sleeping"
+            "state": "sleeping",
+            "num_threads": 42
         }
     },
     "user": {
         "name": "alexk"
     }
-}
+}

metricbeat/module/system/process/_meta/fields.yml

@@ -24,6 +24,10 @@
       type: alias
       path: process.pgid
       migration: true
+    - name: num_threads
+      type: integer
+      description: >
+        Number of threads in the process
     - name: cmdline
       type: keyword
       description: >
@@ -345,7 +349,7 @@
             - name: stats.user.pct
               type: scaled_float
               description: time the cgroup spent in user space, as a percentage of total CPU time
-            
+
             - name: stats.user.norm.pct
               type: scaled_float
               description: time the cgroup spent in user space, as a percentage of total CPU time, normalized by CPU count.
@@ -419,7 +423,7 @@
               type: long
               format: bytes
               description: memory max threshhold
-            
+
             - name: mem.events
               type: group
               description: number of times the controller tripped a given usage level

metricbeat/module/system/process/process.go

@@ -32,7 +32,7 @@ import (
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
 )
 
-var debugf = logp.MakeDebug("system.process")
+var debugf = logp.NewLogger("system.process").Debugf
 
 func init() {
 	mb.Registry.MustAddMetricSet("system", "process", New,

metricbeat/module/system/process/process_test.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 	_ "github.com/elastic/beats/v7/metricbeat/module/system"
@@ -32,7 +33,9 @@ import (
 )
 
 func TestFetch(t *testing.T) {
-	logp.DevelopmentSetup()
+	err := logp.DevelopmentSetup()
+	require.NoError(t, err, "could not set the logger to DevelopmentSetup")
+
 	f := mbtest.NewReportingMetricSetV2Error(t, getConfig())
 	events, errs := mbtest.ReportingFetchV2Error(f)
 	assert.Empty(t, errs)
@@ -44,6 +47,7 @@ func TestFetch(t *testing.T) {
 	assert.Empty(t, errs)
 	assert.NotEmpty(t, events)
 
+	t.Logf("fetched %d events, showing events[0]:", len(events))
 	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
 		events[0].BeatEvent("system", "process").Fields.StringToPrint())
 }

metricbeat/module/system/test_system.py

@@ -112,7 +112,7 @@
 # for some kernel level processes. fd is also part of the system process, but
 # is not available on all OSes and requires root to read for all processes.
 # cgroup is only available on linux.
-SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state"]
+SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state", "num_threads"]
 
 
 class Test(metricbeat.BaseTest):
@@ -439,7 +439,7 @@ def test_process(self):
     @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os")
     def test_process_unix(self):
         """
-        Test system/process output for fields specific of unix systems.
+        Test system/process output checking it has got all expected fields specific of unix systems and no extra ones.
         """
 
         self.render_config_template(