[Metricbeat] K8s apiserver metricset to new data generation (elastic#…

…11113)

This moves the kubernetes/apiserver to the new testing methodology. I kept the old tests around for now but they should be removed as soon as we agree the output of this change is as we expected.

metricbeat/docs/modules/kubernetes/apiserver.asciidoc

@@ -13,3 +13,9 @@ include::../../../module/kubernetes/apiserver/_meta/docs.asciidoc[]
 For a description of each field in the metricset, see the
 <<exported-fields-kubernetes,exported fields>> section.
 
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/kubernetes/apiserver/_meta/data.json[]
+----

metricbeat/mb/testing/data_test.go

@@ -26,9 +26,12 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 
+	"github.com/mitchellh/hashstructure"
+
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v2"
 
@@ -39,6 +42,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/couchbase/cluster"
 	_ "github.com/elastic/beats/metricbeat/module/couchbase/node"
 	_ "github.com/elastic/beats/metricbeat/module/kibana/status"
+	_ "github.com/elastic/beats/metricbeat/module/kubernetes/apiserver"
 	_ "github.com/elastic/beats/metricbeat/module/php_fpm/pool"
 	_ "github.com/elastic/beats/metricbeat/module/php_fpm/process"
 	_ "github.com/elastic/beats/metricbeat/module/rabbitmq/connection"
@@ -55,8 +59,9 @@ var (
 )
 
 type Config struct {
-	Type string
-	URL  string
+	Type   string
+	URL    string
+	Suffix string
 }
 
 func TestAll(t *testing.T) {
@@ -79,13 +84,17 @@ func TestAll(t *testing.T) {
 			log.Fatalf("Unmarshal: %v", err)
 		}
 
-		getTestdataFiles(t, config.URL, moduleName, metricSetName)
+		if config.Suffix == "" {
+			config.Suffix = "json"
+		}
+
+		getTestdataFiles(t, config.URL, moduleName, metricSetName, config.Suffix)
 	}
 }
 
-func getTestdataFiles(t *testing.T, url, module, metricSet string) {
+func getTestdataFiles(t *testing.T, url, module, metricSet, suffix string) {
 
-	ff, _ := filepath.Glob(getMetricsetPath(module, metricSet) + "/_meta/testdata/*.json")
+	ff, _ := filepath.Glob(getMetricsetPath(module, metricSet) + "/_meta/testdata/*." + suffix)
 	var files []string
 	for _, f := range ff {
 		// Exclude all the expected files
@@ -97,12 +106,12 @@ func getTestdataFiles(t *testing.T, url, module, metricSet string) {
 
 	for _, f := range files {
 		t.Run(f, func(t *testing.T) {
-			runTest(t, f, module, metricSet, url)
+			runTest(t, f, module, metricSet, url, suffix)
 		})
 	}
 }
 
-func runTest(t *testing.T, file string, module, metricSetName, url string) {
+func runTest(t *testing.T, file string, module, metricSetName, url, suffix string) {
 
 	// starts a server serving the given file under the given url
 	s := server(t, file, url)
@@ -139,6 +148,13 @@ func runTest(t *testing.T, file string, module, metricSetName, url string) {
 		data = append(data, beatEvent.Fields)
 	}
 
+	// Sorting the events is necessary as events are not necessarily sent in the same order
+	sort.SliceStable(data, func(i, j int) bool {
+		h1, _ := hashstructure.Hash(data[i], nil)
+		h2, _ := hashstructure.Hash(data[j], nil)
+		return h1 < h2
+	})
+
 	output, err := json.MarshalIndent(&data, "", "    ")
 	if err != nil {
 		t.Fatal(err)
@@ -159,7 +175,7 @@ func runTest(t *testing.T, file string, module, metricSetName, url string) {
 
 	assert.Equal(t, string(expected), string(output))
 
-	if strings.HasSuffix(file, "docs.json") {
+	if strings.HasSuffix(file, "docs."+suffix) {
 		writeDataJSON(t, data[0], module, metricSetName)
 	}
 }

metricbeat/module/kubernetes/apiserver/_meta/data.json

@@ -0,0 +1,26 @@
+{
+    "@timestamp": "2019-03-01T08:05:34.853Z",
+    "event": {
+        "dataset": "kubernetes.apiserver",
+        "duration": 115000,
+        "module": "kubernetes"
+    },
+    "kubernetes": {
+        "apiserver": {
+            "request": {
+                "client": "kubectl/v1.6.1 (linux/amd64) kubernetes/b0b7a32",
+                "count": 4,
+                "resource": "configmaps",
+                "scope": "namespace",
+                "verb": "DELETE"
+            }
+        }
+    },
+    "metricset": {
+        "name": "apiserver"
+    },
+    "service": {
+        "address": "127.0.0.1:55555",
+        "type": "kubernetes"
+    }
+}

metricbeat/module/kubernetes/apiserver/_meta/testdata/config.yml

@@ -0,0 +1,3 @@
+type: http
+url: "/metrics"
+suffix: plain