Cherry-pick #8335 to 6.x: Adding elasticsearch/ccr metricset (#8357)

* Adding elasticsearch/ccr metricset (#8335)

This PR adds the `elasticsearch/ccr` metricset which collects cross-cluster replication (CCR) stats from the `GET _ccr/stats` Elasticsearch API and indexes them in `metricbeat-*`.

(cherry picked from commit 0f2b29a)

* Cleaning up CHANGELOG

Author: ycombinator

CHANGELOG.asciidoc

@@ -109,9 +109,8 @@ https://github.com/elastic/beats/compare/v6.4.0...6.x[Check the HEAD diff]
 - Add support to renamed fields planned for redis 5.0. {pull}8167[8167]
 - Allow TCP helper to support delimiters and graphite module to accept multiple metrics in a single payload. {pull}8278[8278]
 - Added 'died' PID state to process_system metricset on system module {pull}8275[8275]
-
-
 - Add `metrics` metricset to MongoDB module. {pull}7611[7611]
+- Added `ccr` metricset to Elasticsearch module. {pull}8335[8335]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -3459,6 +3459,85 @@ type: keyword
 Elasticsearch state id.
 
 
+--
+
+[float]
+== ccr fields
+
+Cross-cluster replication stats
+
+
+
+
+*`elasticsearch.ccr.leader.index`*::
++
+--
+type: keyword
+
+Name of leader index
+
+
+--
+
+*`elasticsearch.ccr.leader.max_seq_no`*::
++
+--
+type: long
+
+Maximum sequence number of operation on the leader shard
+
+
+--
+
+
+*`elasticsearch.ccr.follower.index`*::
++
+--
+type: keyword
+
+Name of follower index
+
+
+--
+
+*`elasticsearch.ccr.follower.shard.number`*::
++
+--
+type: long
+
+Number of the shard within the index
+
+
+--
+
+*`elasticsearch.ccr.follower.operations_indexed`*::
++
+--
+type: long
+
+Number of operations indexed (replicated) into the follower shard from the leader shard
+
+
+--
+
+*`elasticsearch.ccr.follower.time_since_last_fetch.ms`*::
++
+--
+type: long
+
+Time, in ms, since the follower last fetched from the leader
+
+
+--
+
+*`elasticsearch.ccr.follower.global_checkpoint`*::
++
+--
+type: long
+
+Global checkpoint value on follower shard
+
+
 --
 
 [float]

metricbeat/docs/modules/elasticsearch.asciidoc

@@ -51,6 +51,8 @@ It also supports the options described in <<module-http-config-options>>.
 
 The following metricsets are available:
 
+* <<metricbeat-metricset-elasticsearch-ccr,ccr>>
+
 * <<metricbeat-metricset-elasticsearch-cluster_stats,cluster_stats>>
 
 * <<metricbeat-metricset-elasticsearch-index,index>>
@@ -69,6 +71,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-elasticsearch-shard,shard>>
 
+include::elasticsearch/ccr.asciidoc[]
+
 include::elasticsearch/cluster_stats.asciidoc[]
 
 include::elasticsearch/index.asciidoc[]

metricbeat/docs/modules/elasticsearch/ccr.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-elasticsearch-ccr]]
+=== Elasticsearch ccr metricset
+
+beta[]
+
+include::../../../module/elasticsearch/ccr/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-elasticsearch,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/elasticsearch/ccr/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -33,7 +33,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-dropwizard,Dropwizard>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-dropwizard-collector,collector>> beta[]  
 |<<metricbeat-module-elasticsearch,Elasticsearch>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.9+| .9+|  |<<metricbeat-metricset-elasticsearch-cluster_stats,cluster_stats>> beta[]  
+.10+| .10+|  |<<metricbeat-metricset-elasticsearch-ccr,ccr>> beta[]  
+|<<metricbeat-metricset-elasticsearch-cluster_stats,cluster_stats>> beta[]  
 |<<metricbeat-metricset-elasticsearch-index,index>> beta[]  
 |<<metricbeat-metricset-elasticsearch-index_recovery,index_recovery>> beta[]  
 |<<metricbeat-metricset-elasticsearch-index_summary,index_summary>> beta[]  

metricbeat/helper/elastic/elastic.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/metricbeat/mb"
 )
 
@@ -92,3 +93,18 @@ func ReportErrorForMissingField(field string, product Product, r mb.ReporterV2)
 func MakeErrorForMissingField(field string, product Product) error {
 	return fmt.Errorf("Could not find field '%v' in %v stats API response", field, strings.Title(product.String()))
 }
+
+// IsFeatureAvailable returns whether a feature is available in the current product version
+func IsFeatureAvailable(currentProductVersion, featureAvailableInProductVersion string) (bool, error) {
+	currentVersion, err := common.NewVersion(currentProductVersion)
+	if err != nil {
+		return false, err
+	}
+
+	wantVersion, err := common.NewVersion(featureAvailableInProductVersion)
+	if err != nil {
+		return false, err
+	}
+
+	return !currentVersion.LessThan(wantVersion), nil
+}

metricbeat/include/list.go

@@ -0,0 +1,39 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "elasticsearch": {
+        "ccr": {
+            "follower": {
+                "global_checkpoint": 1,
+                "index": "my_follower",
+                "operations_indexed": 2,
+                "shard": {
+                    "number": 0
+                },
+                "time_since_last_fetch": {
+                    "ms": 4926
+                }
+            },
+            "leader": {
+                "index": "my_leader",
+                "max_seq_no": 1
+            }
+        },
+        "cluster": {
+            "id": "KSGkOjOuSg6whAgtpPyhQw",
+            "name": "elasticsearch"
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:9200",
+        "module": "elasticsearch",
+        "name": "ccr",
+        "rtt": 115
+    },
+    "service": {
+        "name": "elasticsearch"
+    }
+}

metricbeat/module/elasticsearch/ccr/_meta/data.json

@@ -0,0 +1,3 @@
+This is the `ccr` metricset of the Elasticsearch module. It interrogates the
+Cross Cluster Replication Stats API endpoint to fetch information about shards
+in the Elasticsearch cluster that are participating in cross-cluster replication.

metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc

@@ -0,0 +1,40 @@
+- name: ccr
+  type: group
+  description: >
+    Cross-cluster replication stats
+  release: beta
+  fields:
+    - name: leader
+      type: group
+      fields:
+        - name: index
+          type: keyword
+          description: >
+            Name of leader index
+        - name: max_seq_no
+          type: long
+          description: >
+            Maximum sequence number of operation on the leader shard
+    - name: follower
+      type: group
+      fields:
+        - name: index
+          type: keyword
+          description: >
+            Name of follower index
+        - name: shard.number
+          type: long
+          description: >
+            Number of the shard within the index
+        - name: operations_indexed
+          type: long
+          description: >
+            Number of operations indexed (replicated) into the follower shard from the leader shard
+        - name: time_since_last_fetch.ms
+          type: long
+          description: >
+            Time, in ms, since the follower last fetched from the leader
+        - name: global_checkpoint
+          type: long
+          description: >
+            Global checkpoint value on follower shard