Merge pull request grafana#146 from grafana/main

Update from upstream repository
periklis · May 26, 2023 · 2772444 · 2772444
2 parents b4291c8 + bb96326
commit 2772444
Show file tree

Hide file tree

Showing 278 changed files with 7,232 additions and 65,358 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -11,3 +11,4 @@ Fixes #<issue number>
 - [ ] Tests updated
 - [ ] `CHANGELOG.md` updated
 - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/upgrading/_index.md`
+- [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213)
diff --git a/.github/workflows/helm-ci.yml b/.github/workflows/helm-ci.yml
@@ -74,7 +74,7 @@ jobs:
         run: ct lint --config "${CT_CONFIGFILE}" --check-version-increment=false
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.5.0
+        uses: helm/kind-action@v1.7.0
         if: steps.list-changed.outputs.changed == 'true'
 
       - name: Install prometheus operator

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 
 ##### Enhancements
 
+* [9515](https://github.com/grafana/loki/pull/9515) **MichelHollands**: Fix String() on vector aggregation LogQL expressions that contain `without ()`.
 * [8067](https://github.com/grafana/loki/pull/8067) **DylanGuedes**: Distributor: Add auto-forget unhealthy members support.
 * [9175](https://github.com/grafana/loki/pull/9175) **MichelHollands**: Ingester: update the `prepare_shutdown` endpoint so it supports GET and DELETE and stores the state on disk.
 * [8953](https://github.com/grafana/loki/pull/8953) **dannykopping**: Querier: block queries by hash.
@@ -27,7 +28,9 @@
 * [8732](https://github.com/grafana/loki/pull/8732) **abaguas**: azure: respect retry config before cancelling the context
 * [9206](https://github.com/grafana/loki/pull/9206) **dannykopping**: Ruler: log rule evaluation detail.
 * [9184](https://github.com/grafana/loki/pull/9184) **periklis**: Bump dskit to introduce IPv6 support for memberlist
+* [9357](https://github.com/grafana/loki/pull/9357) **Indransh**: Add HTTP API to change the log level at runtime
 * [9431](https://github.com/grafana/loki/pull/9431) **dannykopping**: Add more buckets to `loki_memcache_request_duration_seconds` metric; latencies can increase if using memcached with NVMe
+* [8684](https://github.com/grafana/loki/pull/8684) **oleksii-boiko-ua**: Helm: Add hpa templates for read, write and backend components.
 
 ##### Fixes
 

diff --git a/Makefile b/Makefile
@@ -1,3 +1,9 @@
+# Adapted from https://www.thapaliya.com/en/writings/well-documented-makefiles/
+.PHONY: help
+help: ## Display this help and any documented user-facing targets. Other undocumented targets may be present in the Makefile.
+help:
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make <target>\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+
 .DEFAULT_GOAL := all
 .PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos touch-protobuf-sources
 .PHONY: format check-format
@@ -114,7 +120,7 @@ binfmt:
 ################
 # Main Targets #
 ################
-all: promtail logcli loki loki-canary
+all: promtail logcli loki loki-canary ## build all executables (loki, logcli, promtail, loki-canary)
 
 # This is really a check for the CI to make sure generated files are built and checked in manually
 check-generated-files: yacc ragel fmt-proto protos clients/pkg/promtail/server/ui/assets_vfsdata.go
@@ -130,10 +136,10 @@ check-generated-files: yacc ragel fmt-proto protos clients/pkg/promtail/server/u
 # Logcli #
 ##########
 .PHONY: cmd/logcli/logcli
-logcli: cmd/logcli/logcli
-logcli-debug: cmd/logcli/logcli-debug
+logcli: cmd/logcli/logcli ## build logcli executable
+logcli-debug: cmd/logcli/logcli-debug ## build debug logcli executable
 
-logcli-image:
+logcli-image: ## build logcli docker image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) -f cmd/logcli/Dockerfile .
 
 cmd/logcli/logcli:
@@ -145,8 +151,8 @@ cmd/logcli/logcli-debug:
 # Loki #
 ########
 .PHONY: cmd/loki/loki cmd/loki/loki-debug
-loki: cmd/loki/loki
-loki-debug: cmd/loki/loki-debug
+loki: cmd/loki/loki ## build loki executable
+loki-debug: cmd/loki/loki-debug ## build loki debug executable
 
 cmd/loki/loki:
 	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
@@ -158,7 +164,7 @@ cmd/loki/loki-debug:
 # Loki-Canary #
 ###############
 .PHONY: cmd/loki-canary/loki-canary
-loki-canary: cmd/loki-canary/loki-canary
+loki-canary: cmd/loki-canary/loki-canary ## build loki-canary executable
 
 cmd/loki-canary/loki-canary:
 	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
@@ -167,20 +173,20 @@ cmd/loki-canary/loki-canary:
 # Helm #
 ###############
 .PHONY: production/helm/loki/src/helm-test/helm-test
-helm-test: production/helm/loki/src/helm-test/helm-test
+helm-test: production/helm/loki/src/helm-test/helm-test ## run helm tests
 
 # Package Helm tests but do not run them.
 production/helm/loki/src/helm-test/helm-test:
 	CGO_ENABLED=0 go test $(GO_FLAGS) --tags=helm_test -c -o $@ ./$(@D)
 
-helm-lint:
+helm-lint: ## run helm linter
 	$(MAKE) -BC production/helm/loki lint
 
 #################
 # Loki-QueryTee #
 #################
 .PHONY: cmd/querytee/querytee
-loki-querytee: cmd/querytee/querytee
+loki-querytee: cmd/querytee/querytee ## build loki-querytee executable
 
 cmd/querytee/querytee:
 	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
@@ -207,8 +213,8 @@ ifeq ($(PROMTAIL_JOURNAL_ENABLED), true)
 PROMTAIL_GO_TAGS = promtail_journal_enabled
 endif
 .PHONY: clients/cmd/promtail/promtail clients/cmd/promtail/promtail-debug
-promtail: clients/cmd/promtail/promtail
-promtail-debug: clients/cmd/promtail/promtail-debug
+promtail: clients/cmd/promtail/promtail ## build promtail executable
+promtail-debug: clients/cmd/promtail/promtail-debug ## build debug promtail executable
 
 promtail-clean-assets:
 	rm -rf clients/pkg/promtail/server/ui/assets_vfsdata.go
@@ -232,7 +238,7 @@ MIXIN_PATH := production/loki-mixin
 MIXIN_OUT_PATH := production/loki-mixin-compiled
 MIXIN_OUT_PATH_SSD := production/loki-mixin-compiled-ssd
 
-loki-mixin:
+loki-mixin: ## compile the loki mixin
 ifeq ($(BUILD_IN_CONTAINER),true)
 	$(SUDO) docker run $(RM) $(TTY) -i \
 		-v $(shell pwd):/src/loki$(MOUNT_FLAGS) \
@@ -247,7 +253,7 @@ else
 	@mixtool generate all --output-alerts $(MIXIN_OUT_PATH_SSD)/alerts.yaml --output-rules $(MIXIN_OUT_PATH_SSD)/rules.yaml --directory $(MIXIN_OUT_PATH_SSD)/dashboards ${MIXIN_PATH}/mixin-ssd.libsonnet
 endif
 
-loki-mixin-check: loki-mixin
+loki-mixin-check: loki-mixin ## check the loki mixin is up to date
 	@echo "Checking diff"
 	@git diff --exit-code -- $(MIXIN_OUT_PATH) || (echo "Please build mixin by running 'make loki-mixin'" && false)
 	@git diff --exit-code -- $(MIXIN_OUT_PATH_SSD) || (echo "Please build mixin by running 'make loki-mixin'" && false)
@@ -289,7 +295,7 @@ publish: packages
 
 # To run this efficiently on your workstation, run this from the root dir:
 # docker run --rm --tty -i -v $(pwd)/.cache:/go/cache -v $(pwd)/.pkg:/go/pkg -v $(pwd):/src/loki grafana/loki-build-image:0.24.1 lint
-lint:
+lint: ## run linters
 	go version
 	golangci-lint version
 	GO111MODULE=on golangci-lint run -v
@@ -299,7 +305,7 @@ lint:
 # Test #
 ########
 
-test: all
+test: all ## run the unit tests
 	$(GOTEST) -covermode=atomic -coverprofile=coverage.txt -p=4 ./... | sed "s:$$: ${DRONE_STEP_NAME} ${DRONE_SOURCE_BRANCH}:" | tee test_results.txt
 
 compare-coverage:
@@ -312,7 +318,7 @@ compare-coverage:
 clean-protos:
 	rm -rf $(PROTO_GOS)
 
-clean:
+clean: ## clean the generated files
 	rm -rf clients/cmd/promtail/promtail
 	rm -rf cmd/loki/loki
 	rm -rf cmd/logcli/logcli
@@ -421,7 +427,7 @@ define build-rootfs
 	docker rmi rootfsimage -f
 endef
 
-docker-driver: docker-driver-clean
+docker-driver: docker-driver-clean ## build the docker-driver executable
 	$(build-rootfs)
 	docker plugin create $(LOKI_DOCKER_DRIVER):$(PLUGIN_TAG)$(PLUGIN_ARCH) clients/cmd/docker-driver
 
@@ -454,13 +460,13 @@ docker-driver-clean:
 #####################
 # fluent-bit plugin #
 #####################
-fluent-bit-plugin:
+fluent-bit-plugin: ## build the fluent-bit plugin
 	go build $(DYN_GO_FLAGS) -buildmode=c-shared -o clients/cmd/fluent-bit/out_grafana_loki.so ./clients/cmd/fluent-bit/
 
-fluent-bit-image:
+fluent-bit-image: ## build the fluent-bit plugin docker image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile .
 
-fluent-bit-push:
+fluent-bit-push: ## push the fluent-bit plugin docker image
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG)
 
 fluent-bit-test: LOKI_URL ?= http://localhost:3100/loki/api/
@@ -472,17 +478,17 @@ fluent-bit-test:
 ##################
 # fluentd plugin #
 ##################
-fluentd-plugin:
+fluentd-plugin: ## build the fluentd plugin
 	$(MAKE) -BC clients/cmd/fluentd $@
 
-fluentd-plugin-push:
+fluentd-plugin-push: ## push the fluentd plugin
 	$(MAKE) -BC clients/cmd/fluentd $@
 
-fluentd-image:
+fluentd-image: ## build the fluentd docker image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluentd/Dockerfile .
 
 fluentd-push:
-fluentd-image-push:
+fluentd-image-push: ## push the fluentd docker image
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
 
 fluentd-test: LOKI_URL ?= http://loki:3100
@@ -492,7 +498,7 @@ fluentd-test:
 ##################
 # logstash plugin #
 ##################
-logstash-image:
+logstash-image: ## build the logstash image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f clients/cmd/logstash/Dockerfile ./
 
 # Send 10 lines to the local Loki instance.
@@ -501,7 +507,7 @@ logstash-push-test-logs:
 	$(SUDO) docker run -e LOKI_URL="$(LOKI_URL)" -v `pwd`/clients/cmd/logstash/loki-test.conf:/home/logstash/loki.conf --rm \
 		$(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f loki.conf
 
-logstash-push:
+logstash-push: ## push the logstash image
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG)
 
 # Enter an env already configure to build and test logstash output plugin.
@@ -543,41 +549,41 @@ define push-image
 endef
 
 # promtail
-promtail-image:
+promtail-image: ## build the promtail docker image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile .
 promtail-image-cross:
 	$(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile.cross .
 
 promtail-debug-image: OCI_PLATFORMS=
-promtail-debug-image:
+promtail-debug-image: ## build the promtail debug docker image
 	$(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG)-debug -f clients/cmd/promtail/Dockerfile.debug .
 
 promtail-push: promtail-image-cross
 	$(call push-image,promtail)
 
 # loki
-loki-image:
+loki-image: ## build the loki docker image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile .
 loki-image-cross:
 	$(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile.cross .
 
 loki-debug-image: OCI_PLATFORMS=
-loki-debug-image:
+loki-debug-image: ## build the debug loki docker image
 	$(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG)-debug -f cmd/loki/Dockerfile.debug .
 
 loki-push: loki-image-cross
 	$(call push-image,loki)
 
 # loki-canary
-loki-canary-image:
+loki-canary-image: ## build the loki canary docker image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile .
 loki-canary-image-cross:
 	$(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile.cross .
 loki-canary-push: loki-canary-image-cross
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG)
-helm-test-image:
+helm-test-image: ## build the helm test image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) -f production/helm/loki/src/helm-test/Dockerfile .
-helm-test-push: helm-test-image
+helm-test-push: helm-test-image ## push the helm test image
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG)
 
 # loki-querytee
@@ -593,17 +599,17 @@ migrate-image:
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/loki-migrate:$(IMAGE_TAG) -f cmd/migrate/Dockerfile .
 
 # LogQL Analyzer
-logql-analyzer-image:
+logql-analyzer-image: ## build the LogQL Analyzer image
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/logql-analyzer:$(IMAGE_TAG) -f cmd/logql-analyzer/Dockerfile .
-logql-analyzer-push: logql-analyzer-image
+logql-analyzer-push: logql-analyzer-image ## push the LogQL Analyzer image
 	$(call push-image,logql-analyzer)
 
 
 # build-image (only amd64)
 build-image: OCI_PLATFORMS=
-build-image:
+build-image: ## build the docker build image
 	$(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG) ./loki-build-image
-build-image-push: build-image
+build-image-push: build-image ## push the docker build image
 ifneq (,$(findstring WIP,$(IMAGE_TAG)))
 	@echo "Cannot push a WIP image, commit changes first"; \
 	false;

diff --git a/cmd/loki-canary/main.go b/cmd/loki-canary/main.go
@@ -111,9 +111,7 @@ func main() {
 	}
 
 	var tlsConfig *tls.Config
-	tc := config.TLSConfig{
-		InsecureSkipVerify: *insecureSkipVerify,
-	}
+	tc := config.TLSConfig{}
 	if *certFile != "" || *keyFile != "" || *caFile != "" {
 		if !*useTLS {
 			_, _ = fmt.Fprintf(os.Stderr, "Must set --tls when specifying client certs\n")
@@ -122,8 +120,18 @@ func main() {
 		tc.CAFile = *caFile
 		tc.CertFile = *certFile
 		tc.KeyFile = *keyFile
-	}
-	if *useTLS {
+		tc.InsecureSkipVerify = *insecureSkipVerify
+
+		var err error
+		tlsConfig, err = config.NewTLSConfig(&tc)
+		if err != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "TLS configuration error: %s\n", err.Error())
+			os.Exit(1)
+		}
+	} else if *useTLS && *insecureSkipVerify {
+		// Case where cert cannot be trusted but we are also not using mTLS.
+		tc.InsecureSkipVerify = *insecureSkipVerify
+
 		var err error
 		tlsConfig, err = config.NewTLSConfig(&tc)
 		if err != nil {

diff --git a/docs/sources/clients/lambda-promtail/_index.md b/docs/sources/clients/lambda-promtail/_index.md
@@ -188,7 +188,7 @@ scrape_configs:
       - source_labels: ['__aws_cloudwatch_log_group']
         target_label: 'log_group'
       # Maps the loadbalancer name into a label called `loadbalancer_name` for use in Loki.
-      - source_label: ['__aws_s3_log_lb']
+      - source_labels: ['__aws_s3_log_lb']
         target_label: 'loadbalancer_name'
 ```
 

diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md
@@ -356,7 +356,7 @@ is restarted to allow it to continue from where it left off.
 ## scrape_configs
 
 The `scrape_configs` block configures how Promtail can scrape logs from a series
-of targets using a specified discovery method:
+of targets using a specified discovery method. Promtail uses the same [Prometheus scrape_configs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). This means if you already own a Prometheus instance, the config will be very similar:
 
 ```yaml
 # Name to identify this scrape config in the Promtail UI.
@@ -1050,7 +1050,7 @@ labels:
   [ <labelname>: <labelvalue> ... ]
 ```
 
-### Available Labels
+#### Available Labels
 
 When Promtail receives GCP logs, various internal labels are made available for [relabeling](#relabel_configs). This depends on the subscription type chosen.
 
@@ -1108,12 +1108,10 @@ connection_string: <string> | default = "range"
   [ <labelname>: <labelvalue> ... ]
 ```
 
-### Available Labels
+#### Available Labels
 
 When Promtail receives Azure Event Hubs messages, various internal labels are made available for [relabeling](#relabel_configs).
 
-**Available Labels:**
-
 - `__azure_event_hubs_category`: The log category of the message when a message is an application log.
 
 The following list of labels is discovered using the Kafka endpoint in Event Hubs.
@@ -1211,7 +1209,7 @@ labels:
 [use_incoming_timestamp: <bool> | default = false]
 ```
 
-**Available Labels:**
+#### Available Labels
 
 The list of labels below are discovered when consuming kafka:
 
@@ -1255,7 +1253,7 @@ use_incoming_timestamp: <bool>
 
 ```
 
-**Available Labels:**
+#### Available Labels
 
 - `__gelf_message_level`: The GELF level as string.
 - `__gelf_message_host`: The host sending the GELF message.