From fc580bb66bd5a930e1d575735087e6fbb863b2a2 Mon Sep 17 00:00:00 2001 From: fktkrt Date: Tue, 1 Sep 2020 16:38:19 +0200 Subject: [PATCH] Add Grafana integration example (#2408) * add basic Grafana integration setup and README.md Signed-off-by: fktkrt * move instructions to README, fix typo Signed-off-by: fktkrt * add prometheus as datasource and hotrod scrape_config Signed-off-by: fktkrt * reorg into folders, add hotROD dashboard, add basic description to README Signed-off-by: fktkrt * rework folder structure, remove duplicated file, accept suggestions Signed-off-by: fktkrt --- examples/grafana-integration/README.md | 55 + .../grafana-integration/docker-compose.yaml | 68 + .../grafana-integration/grafana/dashboard.yml | 11 + .../grafana/datasources.yaml | 18 + .../grafana/hotrod_metrics_logs.json | 2248 +++++++++++++++++ .../prometheus/prometheus.yml | 16 + 6 files changed, 2416 insertions(+) create mode 100644 examples/grafana-integration/README.md create mode 100644 examples/grafana-integration/docker-compose.yaml create mode 100644 examples/grafana-integration/grafana/dashboard.yml create mode 100644 examples/grafana-integration/grafana/datasources.yaml create mode 100644 examples/grafana-integration/grafana/hotrod_metrics_logs.json create mode 100644 examples/grafana-integration/prometheus/prometheus.yml diff --git a/examples/grafana-integration/README.md b/examples/grafana-integration/README.md new file mode 100644 index 00000000000..c09cbc6f592 --- /dev/null +++ b/examples/grafana-integration/README.md @@ -0,0 +1,55 @@ +# Hot R.O.D. - Rides on Demand - Grafana integration + +This example combines the Hot R.O.D. demo application ([examples/hotrod/](../hotrod/)) with Grafana, Loki and Prometheus integration, to demonstrate logs, metrics and traces correlation. + +## Running via `docker-compose` + +### Prerequisites + +* Clone the Jaeger repository `git clone https://github.com/jaegertracing/jaeger.git`, then `cd examples/grafana-integration` + +* All services will log to Loki via the [official Docker driver plugin](https://grafana.com/docs/loki/latest/clients/docker-driver/). +Install the Loki logging plugin for Docker: + +```bash +docker plugin install \ +grafana/loki-docker-driver:latest \ +--alias loki \ +--grant-all-permissions +``` + +### Run the services + +`docker-compose up` + +### Access the services +* HotROD application at http://localhost:8080 +* Grafana UI at http://localhost:3000 + +### Explore with Loki + +Currently the most powerful way to correlate application logs with traces can be performed via Grafana's Explore interface. + +After setting the datasource to Loki, all the log labels become available, and can be easily filtered using [Loki's LogQL query language](https://grafana.com/docs/loki/latest/logql/). + +For example, after selecting the compose project/service under Log labels , errors can be filtered with the following expression: + +``` +{compose-project="grafana-integration"} |= "error" +``` + +which will list the redis timeout events. + +### HotROD - Metrics and logs overview dashboard + +Since the HotROD application can expose its metrics in Prometheus' format, these can be also used during investigation. + +This example includes a dashboard that contains a log panel for the selected services in real time. These can be also filtered by a search field, that provides `grep`-like features. + +There are also panels to display the ratio/percentage of errors in the current timeframe. + +Additionally, there are graphs for each service, visualing the rate of the requests and showing latency percentiles. + +### Clean up + +`docker-compose down` diff --git a/examples/grafana-integration/docker-compose.yaml b/examples/grafana-integration/docker-compose.yaml new file mode 100644 index 00000000000..d83fab69f9c --- /dev/null +++ b/examples/grafana-integration/docker-compose.yaml @@ -0,0 +1,68 @@ +version: '2' + +services: + grafana: + image: grafana/grafana:7.1.4 + ports: + - '3000:3000' + volumes: + - ./grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml + - ./grafana/hotrod_metrics_logs.json:/etc/grafana/provisioning/dashboards/hotrod_metrics_logs.json + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/api/prom/push' + + loki: + image: grafana/loki:master + ports: + - '3100:3100' + command: -config.file=/etc/loki/local-config.yaml + # send Loki traces to Jaeger + environment: + - JAEGER_AGENT_HOST=jaeger + - JAEGER_AGENT_PORT=6831 + - JAEGER_SAMPLER_TYPE=const + - JAEGER_SAMPLER_PARAM=1 + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/api/prom/push' + + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - '6831:6831' + - '16686:16686' + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/api/prom/push' + + hotrod: + image: jaegertracing/example-hotrod:latest + ports: + - '8080:8080' + - '8083:8083' + command: ["-m","prometheus","all"] + environment: + - JAEGER_AGENT_HOST=jaeger + - JAEGER_AGENT_PORT=6831 + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/api/prom/push' + + prometheus: + image: prom/prometheus:v2.19.1 + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - '9090:9090' + command: + - --config.file=/etc/prometheus/prometheus.yml + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/api/prom/push' diff --git a/examples/grafana-integration/grafana/dashboard.yml b/examples/grafana-integration/grafana/dashboard.yml new file mode 100644 index 00000000000..93bae946e8e --- /dev/null +++ b/examples/grafana-integration/grafana/dashboard.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: +- name: 'HotROD' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/examples/grafana-integration/grafana/datasources.yaml b/examples/grafana-integration/grafana/datasources.yaml new file mode 100644 index 00000000000..c865e81a675 --- /dev/null +++ b/examples/grafana-integration/grafana/datasources.yaml @@ -0,0 +1,18 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: true + isDefault: true + - name: Jaeger + type: jaeger + access: browser + url: http://jaeger:16686 + editable: true + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 diff --git a/examples/grafana-integration/grafana/hotrod_metrics_logs.json b/examples/grafana-integration/grafana/hotrod_metrics_logs.json new file mode 100644 index 00000000000..b48e00aeaef --- /dev/null +++ b/examples/grafana-integration/grafana/hotrod_metrics_logs.json @@ -0,0 +1,2248 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 12611, + "graphTooltip": 0, + "id": 1, + "iteration": 1598696501892, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 43, + "panels": [], + "title": "Loki Overview", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPostfix": true, + "colorValue": true, + "colors": [ + "rgb(31, 255, 7)", + "rgb(31, 255, 7)", + "rgb(31, 255, 7)" + ], + "datasource": "Loki", + "description": "Total Count of log lines in the specified time range", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "$$hashKey": "object:65", + "name": "value to text", + "value": 1 + }, + { + "$$hashKey": "object:66", + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.4.3", + "postfix": " lines", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgb(0, 82, 0)", + "full": false, + "lineColor": "rgb(31, 255, 7)", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "{}", + "targets": [ + { + "expr": "sum(count_over_time(({container_name=\"$container_name\"})[$__interval]))", + "hide": false, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "10,50", + "timeFrom": null, + "timeShift": null, + "title": "Total Count of logs", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "$$hashKey": "object:68", + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPostfix": true, + "colorPrefix": false, + "colorValue": true, + "colors": [ + "rgb(222, 15, 43)", + "rgb(222, 15, 43)", + "rgb(222, 15, 43)" + ], + "datasource": "Loki", + "description": "Total Count: of pattern: $searchable_pattern in the specified time range", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "$$hashKey": "object:1630", + "name": "value to text", + "value": 1 + }, + { + "$$hashKey": "object:1631", + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.4.3", + "postfix": " \"$searchable_pattern\"", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgb(105, 34, 43)", + "full": false, + "lineColor": "#C4162A", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "{}", + "targets": [ + { + "expr": "sum(count_over_time(({container_name=\"$container_name\"} |~ \"(?i)$searchable_pattern\")[$__interval]))", + "hide": false, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "10,50", + "timeFrom": null, + "timeShift": null, + "title": "Total Count: of pattern $searchable_pattern", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "$$hashKey": "object:1633", + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "total" + }, + { + "datasource": "Loki", + "description": "Live logs is a like 'tail -f' in a real time", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "expr": "{container_name=\"$container_name\"} |~ \"(?i)$searchable_pattern\" ", + "hide": false, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Live logs", + "type": "logs" + }, + { + "cacheTimeout": null, + "datasource": "Loki", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "0", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "noValue": "0", + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "#C4162A", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 9, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": false + }, + "pluginVersion": "7.1.4", + "targets": [ + { + "expr": "sum(count_over_time(({container_name=\"$container_name\"} |~ \"(?i)$searchable_pattern\")[$__interval])) * 100 / sum(count_over_time(({container_name=\"$container_name\"})[$__interval]))", + "hide": false, + "legendFormat": "%", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Percentage of \"$searchable_pattern\" for specified time", + "type": "gauge" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 37, + "panels": [], + "title": "Frontend", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_frontend_http_requests_total[5m])", + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Frontend HTTP response codes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_frontend_requests_total[5m])", + "interval": "", + "legendFormat": "{{endpoint}} error: {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Frontend requests total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_frontend_request_latency_count[5m])", + "interval": "", + "legendFormat": "{{endpoint}} - error: {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Frontend request latency count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(hotrod_frontend_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(hotrod_frontend_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(hotrod_frontend_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Frontend latency percentiles", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 21, + "panels": [], + "title": "Route", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_route_http_requests_total[5m])", + "interval": "", + "legendFormat": "{{status_code}} {{endpoint }} ", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Route HTTP response codes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_route_requests_total[5m])", + "interval": "", + "legendFormat": "{{endpoint }} {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Route requests total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_route_request_latency_count[5m])", + "interval": "", + "legendFormat": "{{endpoint}} - error: {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Route request latency count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(hotrod_route_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(hotrod_route_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(hotrod_route_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Route latency percentiles", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 19, + "panels": [], + "title": "Customer", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_customer_http_requests_total[5m])", + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Customer HTTP response codes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_customer_requests_total[5m])", + "interval": "", + "legendFormat": "{{endpoint}} {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Customer requests total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_customer_request_latency_count[5m])", + "interval": "", + "legendFormat": "{{endpoint}} - error: {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Customer request latency count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(hotrod_customer_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(hotrod_customer_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(hotrod_customer_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Customer latency percentiles", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 17, + "panels": [], + "title": "Driver", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_driver_http_requests_total[5m])", + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Driver HTTP response codes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_driver_requests_total[5m])", + "interval": "", + "legendFormat": "{{endpoint}} error: {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Driver HTTP response codes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(hotrod_driver_request_latency_count[5m])", + "interval": "", + "legendFormat": "{{endpoint}} - error: {{error}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Driver request latency count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(hotrod_driver_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(hotrod_driver_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(hotrod_driver_request_latency_bucket[5m])) by (le))", + "interval": "", + "legendFormat": "P50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Driver latency percentiles", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "Loki", + "logging", + "prometheus", + "metrics", + "jaeger", + "hotrod" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "grafana-jaeger-integration_hotrod_1", + "value": "grafana-jaeger-integration_hotrod_1" + }, + "datasource": "Loki", + "definition": "label_values({container_name=~\".+\"}, container_name)", + "hide": 0, + "includeAll": false, + "label": "Service", + "multi": false, + "name": "container_name", + "options": [], + "query": "label_values({container_name=~\".+\"}, container_name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "error", + "value": "error" + }, + "hide": 0, + "label": "Search (case insensitive)", + "name": "searchable_pattern", + "options": [ + { + "selected": true, + "text": "error", + "value": "error" + } + ], + "query": "error", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "HotROD - Metrics and Logs Overview", + "uid": "fRIvzUZMz", + "version": 1 +} \ No newline at end of file diff --git a/examples/grafana-integration/prometheus/prometheus.yml b/examples/grafana-integration/prometheus/prometheus.yml new file mode 100644 index 00000000000..6a3c9d84a9c --- /dev/null +++ b/examples/grafana-integration/prometheus/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: + - localhost:9090 + + - job_name: 'hotrod-application' + static_configs: + - targets: + - hotrod:8083 +