Add improved config/event output options to windows/perfmon metricset (…

…elastic#17596)

* config

* work on perfmon

* progress

* work on perf

* Create tests

* work on documentation

* fix test

* changelog

* work on webserver

* fix test

* add test

* validate

* dynamic mapping

* init format

* work on test

(cherry picked from commit 1a50329)

CHANGELOG.next.asciidoc

@@ -404,6 +404,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add test for documented fields check for metricsets without a http input. {issue}17315[17315] {pull}17334[17334]
 - Add final tests and move label to GA for the azure module in metricbeat. {pull}17319[17319]
 - Added documentation for running Metricbeat in Cloud Foundry. {pull}17275[17275]
+- Refactor windows/perfmon metricset configuration options and event output. {pull}17596[17596]
 - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429]
 
 *Packetbeat*

metricbeat/docs/fields.asciidoc

@@ -18569,20 +18569,13 @@ type: object
 
 --
 
-[float]
-=== website
-
-website
-
-
-
-*`iis.website.name`*::
+*`iis.website.*.*`*::
 +
 --
-website name
+website
 
 
-type: keyword
+type: object
 
 --
 
@@ -38902,6 +38895,33 @@ Module for Windows
 
 
 
+[float]
+=== perfmon
+
+perfmon
+
+
+
+*`windows.perfmon.instance`*::
++
+--
+Instance value.
+
+
+type: keyword
+
+--
+
+*`windows.perfmon.metrics.*.*`*::
++
+--
+Metric values returned.
+
+
+type: object
+
+--
+
 [float]
 === service
 

metricbeat/docs/modules/windows.asciidoc

@@ -5,8 +5,13 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-module-windows]]
 == Windows module
 
-This is the Windows module. It collects metrics from Windows systems,
-by default metricset `service` is enabled.
+This is the `windows` module which collects metrics from Windows systems.
+The module contains the `service` metricset, which is set up by default when the `windows` module is enabled.
+The `service` metricset will retrieve status information of the services on the Windows machines. The second `windows`
+metricset is `perfmon` which collects Windows performance counter values.
+
+
+
 
 
 [float]
@@ -24,11 +29,14 @@ metricbeat.modules:
   period: 10s
   perfmon.ignore_non_existent_counters: false
   perfmon.group_measurements_by_instance: false
-  perfmon.counters:
-  #  - instance_label: processor.name
-  #    instance_name: total
-  #    measurement_label: processor.time.total.pct
-  #    query: '\Processor Information(_Total)\% Processor Time'
+  perfmon.queries:
+#  - object: 'Process'
+#    instance: ["*"]
+#    counters:
+#    - name: 'Disk Writes/sec'
+#      field: physical_disk.write.per_sec
+#      format: "float"
+#    - name: "% Disk Write Time"
 
 - module: windows
   metricsets: ["service"]

metricbeat/metricbeat.reference.yml

@@ -848,11 +848,14 @@ metricbeat.modules:
   period: 10s
   perfmon.ignore_non_existent_counters: false
   perfmon.group_measurements_by_instance: false
-  perfmon.counters:
-  #  - instance_label: processor.name
-  #    instance_name: total
-  #    measurement_label: processor.time.total.pct
-  #    query: '\Processor Information(_Total)\% Processor Time'
+  perfmon.queries:
+#  - object: 'Process'
+#    instance: ["*"]
+#    counters:
+#    - name: 'Disk Writes/sec'
+#      field: physical_disk.write.per_sec
+#      format: "float"
+#    - name: "% Disk Write Time"
 
 - module: windows
   metricsets: ["service"]

metricbeat/module/windows/_meta/config.reference.yml

@@ -4,11 +4,14 @@
   period: 10s
   perfmon.ignore_non_existent_counters: false
   perfmon.group_measurements_by_instance: false
-  perfmon.counters:
-  #  - instance_label: processor.name
-  #    instance_name: total
-  #    measurement_label: processor.time.total.pct
-  #    query: '\Processor Information(_Total)\% Processor Time'
+  perfmon.queries:
+#  - object: 'Process'
+#    instance: ["*"]
+#    counters:
+#    - name: 'Disk Writes/sec'
+#      field: physical_disk.write.per_sec
+#      format: "float"
+#    - name: "% Disk Write Time"
 
 - module: windows
   metricsets: ["service"]

metricbeat/module/windows/_meta/config.yml

@@ -1,22 +1,17 @@
 - module: windows
-  #metricsets:
-  #  - service
+  metricsets:
+    - service
   period: 1m
 
 #- module: windows
 #  metricsets:
-#    - perfmon
+#  - perfmon
 #  period: 10s
-#  perfmon.counters:
-#    - instance_label: processor.name
-#      instance_name: total
-#      measurement_label: processor.time.total.pct
-#      query: '\Processor Information(_Total)\% Processor Time'
-#
-#    - instance_label: physical_disk.name
-#      measurement_label: physical_disk.write.per_sec
-#      query: '\PhysicalDisk(*)\Disk Writes/sec'
-#
-#    - instance_label: physical_disk.name
-#      measurement_label: physical_disk.write.time.pct
-#      query: '\PhysicalDisk(*)\% Disk Write Time'
+#  perfmon.queries:
+#   - object: 'Process'
+#     instance: ["*"]
+#     counters:
+#     - name: 'Disk Writes/sec'
+#       field: physical_disk.write.per_sec
+#       format: "float"
+#     - name: "% Disk Write Time"

metricbeat/module/windows/_meta/docs.asciidoc

@@ -1,2 +1,7 @@
-This is the Windows module. It collects metrics from Windows systems,
-by default metricset `service` is enabled.
+This is the `windows` module which collects metrics from Windows systems.
+The module contains the `service` metricset, which is set up by default when the `windows` module is enabled.
+The `service` metricset will retrieve status information of the services on the Windows machines. The second `windows`
+metricset is `perfmon` which collects Windows performance counter values.
+
+
+

metricbeat/module/windows/perfmon/_meta/data.json

@@ -1,24 +1,30 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
+    "event": {
+        "dataset": "windows.perfmon",
+        "duration": 115000,
+        "module": "windows"
     },
     "metricset": {
-        "module": "windows",
         "name": "perfmon",
-        "rtt": 115
+        "period": 10000
+    },
+    "service": {
+        "type": "windows"
     },
     "windows": {
         "perfmon": {
-            "processor": {
-                "name": "_Total",
-                "time": {
-                    "total": {
-                        "pct": 1.4663385364361736
+            "instance": "_Total",
+            "metrics": {
+                "processor": {
+                    "time": {
+                        "total": {
+                            "pct": 6.310940413107646
+                        }
                     }
                 }
-            }
+            },
+            "object": "Processor Information"
         }
     }
 }

metricbeat/module/windows/perfmon/_meta/docs.asciidoc

@@ -15,6 +15,28 @@ to collect. The example below collects processor time and disk writes every
   period: 10s
   perfmon.ignore_non_existent_counters: true
   perfmon.group_measurements_by_instance: true
+  perfmon.queries:
+  - object: "Process"
+    instance: ["svchost*", "conhost*"]
+    counters:
+    - name: "% Processor Time"
+      field: time.processor.pct
+      format: "float"
+    - name: "Thread Count"
+      field: thread_count
+    - name: "IO Read Operations/sec"
+  - object: "PhysicalDisk"
+    field : "disk"
+    instance: "*"
+    counters:
+    - name: "Disk Writes/sec"
+    - name: "% Disk Write Time"
+      field: "write_time"
+      format: "float"
+
+
+  // deprecated, will be removed in 8.0
+
   perfmon.counters:
     - instance_label: processor.name
       instance_name: total
@@ -46,7 +68,33 @@ counter requires three config options - `instance_label`, `measurement_label`,
 and `query`.
 
 [float]
-==== Counter Configuration
+==== Query Configuration
+
+Each item in the `query` list specifies multiple perfmon queries to perform. In the
+events generated by the metricset these configuration options map to the field
+values as shown below.
+
+*`object`*:: The performance object to query. A performance object can be a physical component, such as processors, disks, and memory, or a system object, such as processes and threads. Required
+
+*`field`*:: The object field/label. Not required, if not entered, it will be `object`.
+
+*`instance`*:: Matches the ParentInstance, ObjectInstance, and InstanceIndex are included in the path if multiple instances of the object can exist. Not required for performance counters which do not contain one.
+
+*`counters`*:: List of the partial counter paths (At least one partial counter path is required).
+
+*`name`*:: The counter name. Required. This is the counter specified in Performance Data Helper (PDH) syntax. For example in case of the counter path `\Processor Information(_Total)\% Processor Time`,
+the value for this configuration option will be `% Processor Time`.
+
+*`field`*:: The counter path value field/label. Not required, if not entered, it will be generated based on the counter path.
+
+*`format`*:: Format of the measurement value. The value can be either `float`, `large` or
+`long`. The default is `float`.
+
+
+
+
+[float]
+==== Deprecated Counter Configuration
 
 Each item in the `counters` list specifies a perfmon query to perform. In the
 events generated by the metricset these configuration options map to the field
@@ -77,6 +125,6 @@ Performance Data Helper (PDH) syntax. This field is required. For example
 place of an instance name to perform a wildcard query that generates an event
 for each counter instance (e.g. `\PhysicalDisk(*)\Disk Writes/sec`).
 
-*`format`*:: Format of the measurement value. The value can be either `float` or
+*`format`*:: Format of the measurement value. The value can be either `float`, `large` or
 `long`. The default is `float`.
 

metricbeat/module/windows/perfmon/_meta/fields.yml

@@ -1 +1,17 @@
-- release: beta
+- name: perfmon
+  type: group
+  release: beta
+  description: >
+    perfmon
+  fields:
+  - name: instance
+    type: keyword
+    description: |
+      Instance value.
+  - name: metrics.*.*
+    type: object
+    object_type: float
+    object_type_mapping_type: "*"
+    description: >
+      Metric values returned.
+