Loki mixin: publish compiled version of the mixin

.drone/drone.jsonnet

@@ -413,6 +413,9 @@ local manifest(apps) = pipeline('manifest') {
         image: 'grafana/jsonnet-build:c8b75df',
         depends_on: ['clone'],
       },
+      make('loki-mixin-check', container=false) {
+        depends_on: ['clone'],
+      },
     ],
   },
 ] + [

.drone/drone.yml

@@ -41,14 +41,14 @@ steps:
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: check-drone-drift
 - commands:
   - make BUILD_IN_CONTAINER=false check-generated-files
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: check-generated-files
 - commands:
   - cd ..
@@ -57,23 +57,23 @@ steps:
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: clone-main
 - commands:
   - make BUILD_IN_CONTAINER=false test
   depends_on:
   - clone
   - clone-main
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: test
 - commands:
   - cd ../loki-main
   - BUILD_IN_CONTAINER=false make test
   depends_on:
   - clone-main
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: test-main
 - commands:
   - make BUILD_IN_CONTAINER=false compare-coverage old=../loki-main/test_results.txt
@@ -83,7 +83,7 @@ steps:
   - test
   - test-main
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: compare-coverage
 - commands:
   - pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')
@@ -96,15 +96,15 @@ steps:
     TOKEN:
       from_secret: github_token
     USER: grafanabot
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: report-coverage
 - commands:
   - make BUILD_IN_CONTAINER=false lint
   depends_on:
   - clone
   - check-generated-files
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: lint
 - commands:
   - make BUILD_IN_CONTAINER=false check-mod
@@ -113,7 +113,7 @@ steps:
   - test
   - lint
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: check-mod
 - commands:
   - apk add make bash && make lint-scripts
@@ -125,21 +125,21 @@ steps:
   - clone
   - check-generated-files
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: loki
 - commands:
   - make BUILD_IN_CONTAINER=false validate-example-configs
   depends_on:
   - loki
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: validate-example-configs
 - commands:
   - make BUILD_IN_CONTAINER=false check-example-config-doc
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.20.4
+  image: grafana/loki-build-image:0.21.0
   name: check-example-config-doc
 trigger:
   event:
@@ -160,6 +160,13 @@ steps:
   environment: {}
   image: grafana/jsonnet-build:c8b75df
   name: lint-jsonnet
+- commands:
+  - make BUILD_IN_CONTAINER=false loki-mixin-check
+  depends_on:
+  - clone
+  environment: {}
+  image: grafana/loki-build-image:0.21.0
+  name: loki-mixin-check
 trigger:
   event:
   - push
@@ -1167,6 +1174,6 @@ kind: secret
 name: deploy_config
 ---
 kind: signature
-hmac: cb6f2957ccaf19841b99a76ec820fe1d3b32df4f20ea34b651afda4d27c9f1a6
+hmac: cd2881f8ac64b8f4152e9170d02a526b2fb21d10e7ef1a036cd7ecb8d7ea8951
 
 ...

Makefile

@@ -27,7 +27,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
 BUILD_IN_CONTAINER ?= true
 
 # ensure you run `make drone` after changing this
-BUILD_IMAGE_VERSION := 0.20.4
+BUILD_IMAGE_VERSION := 0.21.0
 
 # Docker image info
 IMAGE_PREFIX ?= grafana
@@ -210,6 +210,28 @@ clients/cmd/promtail/promtail-debug:
 	CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_DEBUG_GO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
+#########
+# Mixin #
+#########
+
+MIXIN_PATH := production/loki-mixin
+MIXIN_OUT_PATH := production/loki-mixin-compiled
+
+loki-mixin:
+ifeq ($(BUILD_IN_CONTAINER),true)
+	$(SUDO) docker run $(RM) $(TTY) -i \
+		-v $(shell pwd):/src/loki$(MOUNT_FLAGS) \
+		$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@;
+else
+	@rm -rf $(MIXIN_OUT_PATH) && mkdir $(MIXIN_OUT_PATH)
+	@cd $(MIXIN_PATH) && jb install
+	@mixtool generate all --output-alerts $(MIXIN_OUT_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_PATH)/rules.yaml --directory $(MIXIN_OUT_PATH)/dashboards ${MIXIN_PATH}/mixin.libsonnet
+endif
+
+loki-mixin-check: loki-mixin
+	@echo "Checking diff"
+	@git diff --exit-code -- $(MIXIN_OUT_PATH) || (echo "Please build mixin by running 'make loki-mixin'" && false)
+
 ###############
 # Migrate #
 ###############

production/loki-mixin-compiled/alerts.yaml

@@ -0,0 +1,41 @@
+groups:
+- name: loki_alerts
+  rules:
+  - alert: LokiRequestErrors
+    annotations:
+      message: |
+        {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
+    expr: |
+      100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[1m])) by (namespace, job, route)
+        /
+      sum(rate(loki_request_duration_seconds_count[1m])) by (namespace, job, route)
+        > 10
+    for: 15m
+    labels:
+      severity: critical
+  - alert: LokiRequestPanics
+    annotations:
+      message: |
+        {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics.
+    expr: |
+      sum(increase(loki_panic_total[10m])) by (namespace, job) > 0
+    labels:
+      severity: critical
+  - alert: LokiRequestLatency
+    annotations:
+      message: |
+        {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
+    expr: |
+      namespace_job_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*"} > 1
+    for: 15m
+    labels:
+      severity: critical
+  - alert: LokiTooManyCompactorsRunning
+    annotations:
+      message: |
+        {{ $labels.namespace }} has had {{ printf "%.0f" $value }} compactors running for more than 5m. Only one compactor should run at a time.
+    expr: |
+      sum(loki_boltdb_shipper_compactor_running) by (namespace) > 1
+    for: 5m
+    labels:
+      severity: warning