Skip to content

Commit 2e1b7c8

Browse files
ruflinjsoriano
ruflin
authored and
jsoriano
committed
Extract Kibana dashboards (#7224)
Currently when a PR is opened to change or add a dashboard the PR's are hard to review even if only a small detail was changed. The reasons is that the Kibana json objects contain json as string. The content inside these strings is valid JSON if it is decoded. With this PR the dashboards in Metricbeat are modified that they contain the full decoded JSON objects instead of the string. Additional the JSON entries are sorted. This makes also the content of the visualisations readable, will create minimal diffs and allows to even apply small fixes on the code base. In addition we can now validate if it's valid JSON and introduce automatic scripts to remove potentially unneeded fields from the dashboards to clean them up. All the existing dashboards were decoded with the following command: ``` python ../libbeat/scripts/unpack_dashboards.py --transform=decode --glob="/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/metricbeat/module/*/_meta/kibana/6/dashboard/*.json" ``` This command has to be applied to dashboards exported from Kibana before adding them to the module directory. To make sure the import still works as before, on collection of the dashboard they are converted back into the encoded format. This change currently only applies to Metricbeat dashboards but could be applied to all.
1 parent ccd36fb commit 2e1b7c8

File tree

24 files changed

+14570
-2678
lines changed

24 files changed

+14570
-2678
lines changed

‎libbeat/scripts/unpack_dashboards.py

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import json
2+
import sys
3+
import glob
4+
import argparse
5+
6+
7+
def transform_data(data, method):
8+
for obj in data["objects"]:
9+
if "uiStateJSON" in obj["attributes"]:
10+
obj["attributes"]["uiStateJSON"] = method(obj["attributes"]["uiStateJSON"])
11+
12+
if "optionsJSON" in obj["attributes"]:
13+
obj["attributes"]["optionsJSON"] = method(obj["attributes"]["optionsJSON"])
14+
15+
if "panelsJSON" in obj["attributes"]:
16+
obj["attributes"]["panelsJSON"] = method(obj["attributes"]["panelsJSON"])
17+
18+
if "visState" in obj["attributes"]:
19+
obj["attributes"]["visState"] = method(obj["attributes"]["visState"])
20+
21+
if "kibanaSavedObjectMeta" in obj["attributes"] and "searchSourceJSON" in obj["attributes"]["kibanaSavedObjectMeta"]:
22+
obj["attributes"]["kibanaSavedObjectMeta"]["searchSourceJSON"] = method(
23+
obj["attributes"]["kibanaSavedObjectMeta"]["searchSourceJSON"])
24+
25+
26+
def transform_file(path, method):
27+
with open(path) as f:
28+
data = json.load(f)
29+
30+
transform_data(data, method)
31+
return data
32+
33+
34+
if __name__ == "__main__":
35+
36+
parser = argparse.ArgumentParser(description="Convert dashboards")
37+
parser.add_argument("--transform", help="Decode or encode", default="encode")
38+
parser.add_argument("--glob", help="Glob pattern")
39+
40+
args = parser.parse_args()
41+
42+
paths = glob.glob(args.glob)
43+
44+
method = json.dumps
45+
if args.transform == "decode":
46+
method = json.loads
47+
48+
for path in paths:
49+
data = transform_file(path, method)
50+
new_data = json.dumps(data, sort_keys=True, indent=4)
51+
52+
with open(path, 'w') as f:
53+
f.write(new_data)

‎metricbeat/Makefile

+3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ kibana:
2121
@rm -rf _meta/kibana
2222
@mkdir -p _meta/kibana
2323
@-cp -r module/*/_meta/kibana _meta/
24+
@-cp -r module/*/_meta/kibana _meta/
25+
@# Convert all dashboards to string
26+
@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana/6/dashboard/*.json"
2427

2528
# Collects all module docs
2629
.PHONY: collect-docs

‎metricbeat/module/apache/_meta/kibana/6/dashboard/Metricbeat-apache-overview.json

+780-155
Large diffs are not rendered by default.

‎metricbeat/module/docker/_meta/kibana/6/dashboard/Metricbeat-docker-overview.json

+1,015-186
Large diffs are not rendered by default.

‎metricbeat/module/golang/_meta/kibana/6/dashboard/Metricbeat-golang-overview.json

+296-112
Large diffs are not rendered by default.

‎metricbeat/module/haproxy/_meta/kibana/6/dashboard/Metricbeat-haproxy-backend.json

+113-22
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,114 @@
11
{
2-
"objects": [
3-
{
4-
"attributes": {
5-
"description": "",
6-
"hits": 0,
7-
"kibanaSavedObjectMeta": {
8-
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
9-
},
10-
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
11-
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":6,\"h\":2,\"i\":\"1\"},\"id\":\"a64b4fd0-471c-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":6,\"y\":0,\"w\":3,\"h\":2,\"i\":\"2\"},\"id\":\"794b6cd0-471d-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":6,\"y\":2,\"w\":6,\"h\":4,\"i\":\"3\"},\"id\":\"bb0ab500-4735-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"4\"},\"id\":\"40bed190-473b-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"5\",\"gridData\":{\"x\":0,\"y\":2,\"w\":6,\"h\":2,\"i\":\"5\"},\"id\":\"0751ed00-479c-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":0,\"y\":4,\"w\":6,\"h\":2,\"i\":\"6\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"b3463670-47a1-11e8-bc13-1397384faad3\"}]",
12-
"timeRestore": false,
13-
"title": "[Metricbeat Haproxy] Backend",
14-
"version": 1
15-
},
16-
"id": "9151c900-471d-11e8-bc13-1397384faad3",
17-
"type": "dashboard",
18-
"updated_at": "2018-04-24T18:31:25.838Z",
19-
"version": 15
20-
}
21-
],
22-
"version": "6.2.2"
23-
}
2+
"objects": [
3+
{
4+
"attributes": {
5+
"description": "",
6+
"hits": 0,
7+
"kibanaSavedObjectMeta": {
8+
"searchSourceJSON": {
9+
"filter": [],
10+
"highlightAll": true,
11+
"query": {
12+
"language": "lucene",
13+
"query": ""
14+
},
15+
"version": true
16+
}
17+
},
18+
"optionsJSON": {
19+
"darkTheme": false,
20+
"hidePanelTitles": false,
21+
"useMargins": true
22+
},
23+
"panelsJSON": [
24+
{
25+
"gridData": {
26+
"h": 2,
27+
"i": "1",
28+
"w": 6,
29+
"x": 0,
30+
"y": 0
31+
},
32+
"id": "a64b4fd0-471c-11e8-bc13-1397384faad3",
33+
"panelIndex": "1",
34+
"type": "visualization",
35+
"version": "6.2.2"
36+
},
37+
{
38+
"gridData": {
39+
"h": 2,
40+
"i": "2",
41+
"w": 3,
42+
"x": 6,
43+
"y": 0
44+
},
45+
"id": "794b6cd0-471d-11e8-bc13-1397384faad3",
46+
"panelIndex": "2",
47+
"type": "visualization",
48+
"version": "6.2.2"
49+
},
50+
{
51+
"gridData": {
52+
"h": 4,
53+
"i": "3",
54+
"w": 6,
55+
"x": 6,
56+
"y": 2
57+
},
58+
"id": "bb0ab500-4735-11e8-bc13-1397384faad3",
59+
"panelIndex": "3",
60+
"type": "visualization",
61+
"version": "6.2.2"
62+
},
63+
{
64+
"gridData": {
65+
"h": 2,
66+
"i": "4",
67+
"w": 3,
68+
"x": 9,
69+
"y": 0
70+
},
71+
"id": "40bed190-473b-11e8-bc13-1397384faad3",
72+
"panelIndex": "4",
73+
"type": "visualization",
74+
"version": "6.2.2"
75+
},
76+
{
77+
"gridData": {
78+
"h": 2,
79+
"i": "5",
80+
"w": 6,
81+
"x": 0,
82+
"y": 2
83+
},
84+
"id": "0751ed00-479c-11e8-bc13-1397384faad3",
85+
"panelIndex": "5",
86+
"type": "visualization",
87+
"version": "6.2.2"
88+
},
89+
{
90+
"gridData": {
91+
"h": 2,
92+
"i": "6",
93+
"w": 6,
94+
"x": 0,
95+
"y": 4
96+
},
97+
"id": "b3463670-47a1-11e8-bc13-1397384faad3",
98+
"panelIndex": "6",
99+
"type": "visualization",
100+
"version": "6.2.2"
101+
}
102+
],
103+
"timeRestore": false,
104+
"title": "[Metricbeat Haproxy] Backend",
105+
"version": 1
106+
},
107+
"id": "9151c900-471d-11e8-bc13-1397384faad3",
108+
"type": "dashboard",
109+
"updated_at": "2018-04-24T18:31:25.838Z",
110+
"version": 15
111+
}
112+
],
113+
"version": "6.2.2"
114+
}

‎metricbeat/module/haproxy/_meta/kibana/6/dashboard/Metricbeat-haproxy-frontend.json

+61-22
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,62 @@
11
{
2-
"objects": [
3-
{
4-
"attributes": {
5-
"description": "",
6-
"hits": 0,
7-
"kibanaSavedObjectMeta": {
8-
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
9-
},
10-
"optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}",
11-
"panelsJSON": "[{\"panelIndex\":\"2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":6,\"h\":3,\"i\":\"2\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"a64b4fd0-471c-11e8-bc13-1397384faad3\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":6,\"y\":0,\"w\":6,\"h\":3,\"i\":\"3\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"86159190-47c5-11e8-bc13-1397384faad3\"}]",
12-
"timeRestore": false,
13-
"title": "[Metricbeat Haproxy] Frontend",
14-
"version": 1
15-
},
16-
"id": "d5878d00-47c5-11e8-bc13-1397384faad3",
17-
"type": "dashboard",
18-
"updated_at": "2018-04-24T18:32:51.945Z",
19-
"version": 5
20-
}
21-
],
22-
"version": "6.2.2"
23-
}
2+
"objects": [
3+
{
4+
"attributes": {
5+
"description": "",
6+
"hits": 0,
7+
"kibanaSavedObjectMeta": {
8+
"searchSourceJSON": {
9+
"filter": [],
10+
"highlightAll": true,
11+
"query": {
12+
"language": "lucene",
13+
"query": ""
14+
},
15+
"version": true
16+
}
17+
},
18+
"optionsJSON": {
19+
"darkTheme": false,
20+
"hidePanelTitles": false,
21+
"useMargins": true
22+
},
23+
"panelsJSON": [
24+
{
25+
"gridData": {
26+
"h": 3,
27+
"i": "2",
28+
"w": 6,
29+
"x": 0,
30+
"y": 0
31+
},
32+
"id": "a64b4fd0-471c-11e8-bc13-1397384faad3",
33+
"panelIndex": "2",
34+
"type": "visualization",
35+
"version": "6.2.2"
36+
},
37+
{
38+
"gridData": {
39+
"h": 3,
40+
"i": "3",
41+
"w": 6,
42+
"x": 6,
43+
"y": 0
44+
},
45+
"id": "86159190-47c5-11e8-bc13-1397384faad3",
46+
"panelIndex": "3",
47+
"type": "visualization",
48+
"version": "6.2.2"
49+
}
50+
],
51+
"timeRestore": false,
52+
"title": "[Metricbeat Haproxy] Frontend",
53+
"version": 1
54+
},
55+
"id": "d5878d00-47c5-11e8-bc13-1397384faad3",
56+
"type": "dashboard",
57+
"updated_at": "2018-04-24T18:32:51.945Z",
58+
"version": 5
59+
}
60+
],
61+
"version": "6.2.2"
62+
}

0 commit comments

Comments
 (0)