Merge branch 'main' into sparsehistogram

.circleci/config.yml

@@ -2,7 +2,7 @@
 version: 2.1
 
 orbs:
-  prometheus: prometheus/prometheus@0.15.0
+  prometheus: prometheus/prometheus@0.16.0
   go: circleci/go@1.7.0
   win: circleci/windows@2.3.0
 
@@ -12,9 +12,9 @@ executors:
   golang:
     docker:
       - image: quay.io/prometheus/golang-builder:1.17-base
-  golang_115:
+  golang_oldest:
     docker:
-      - image: quay.io/prometheus/golang-builder:1.15-base
+      - image: quay.io/prometheus/golang-builder:1.16-base
 
 jobs:
   test_go:
@@ -37,6 +37,7 @@ jobs:
             GOMAXPROCS: "2"
             GO111MODULE: "on"
       - run: go test ./tsdb/ -test.tsdb-isolation=false
+      - run: make -C documentation/examples/remote_storage
       - prometheus/check_proto:
           version: "3.15.8"
       - prometheus/store_artifact:
@@ -81,18 +82,17 @@ jobs:
           command: refreshenv
       - run:
           command: |
-            $env:GOARCH=""; $env:GOOS=""; cd web/ui; go generate
-            cd ../..
             $TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"}
             go test $TestTargets -vet=off -v
           environment:
             GOGC: "20"
             GOOPTS: "-p 2"
 
-  test_tsdb_go115:
-    executor: golang_115
+  test_golang_oldest:
+    executor: golang_oldest
     steps:
       - checkout
+      - run: make build
       - run: go test ./tsdb/...
       - run: go test ./tsdb/ -test.tsdb-isolation=false
 
@@ -114,7 +114,6 @@ jobs:
     steps:
       - checkout
       - run: ./scripts/sync_repo_files.sh
-      - run: ./scripts/sync_codemirror.sh
 
 workflows:
   version: 2
@@ -128,7 +127,7 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - test_tsdb_go115:
+      - test_golang_oldest:
           filters:
             tags:
               only: /.*/
@@ -179,10 +178,10 @@ workflows:
             branches:
               ignore: /.*/
           image: circleci/golang:1-node
-  nightly:
+  daily:
     triggers:
       - schedule:
-          cron: "0 0 * * *"
+          cron: "49 19 * * *"
           filters:
             branches:
               only:

.github/dependabot.yml

@@ -3,17 +3,16 @@ updates:
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
   - package-ecosystem: "npm"
     directory: "/web/ui"
-    open-pull-requests-limit: 0
     schedule:
-      interval: "weekly"
+      interval: "monthly"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"

.github/workflows/buf-lint.yml

@@ -0,0 +1,20 @@
+name: buf.build
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/buf-lint.yml"
+      - "**.proto"
+jobs:
+  buf:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bufbuild/buf-setup-action@v1.1.0
+      - uses: bufbuild/buf-lint-action@v1
+        with:
+          input: 'prompb'
+      - uses: bufbuild/buf-breaking-action@v1
+        with:
+          input: 'prompb'
+          against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD,subdir=prompb'

.github/workflows/buf.yml

@@ -0,0 +1,23 @@
+name: buf.build
+on:
+  push:
+    branches:
+      - main
+jobs:
+  buf:
+    name: lint and publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bufbuild/buf-setup-action@v1.1.0
+      - uses: bufbuild/buf-lint-action@v1
+        with:
+          input: 'prompb'
+      - uses: bufbuild/buf-breaking-action@v1
+        with:
+          input: 'prompb'
+          against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD~1,subdir=prompb'
+      - uses: bufbuild/buf-push-action@v1
+        with:
+          input: 'prompb'
+          buf_token: ${{ secrets.BUF_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -35,7 +35,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/fuzzing.yml

@@ -1,10 +1,6 @@
 name: CIFuzz
 on:
   pull_request:
-    paths:
-      - "go.sum"
-      - "go.mod"
-      - "**.go"
 jobs:
   Fuzzing:
     runs-on: ubuntu-latest
@@ -22,7 +18,7 @@ jobs:
           fuzz-seconds: 600
           dry-run: false
       - name: Upload Crash
-        uses: actions/upload-artifact@v2.3.0
+        uses: actions/upload-artifact@v2.3.1
         if: failure() && steps.build.outcome == 'success'
         with:
           name: artifacts

.github/workflows/golangci-lint.yml

@@ -9,23 +9,19 @@ on:
       - ".github/workflows/golangci-lint.yml"
       - ".golangci.yml"
   pull_request:
-    paths:
-      - "go.sum"
-      - "go.mod"
-      - "**.go"
-      - "scripts/errcheck_excludes.txt"
-      - ".github/workflows/golangci-lint.yml"
-      - ".golangci.yml"
 
 jobs:
   golangci:
     name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
-
+        uses: actions/checkout@v3
+      - name: install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.x
       - name: Lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3.1.0
         with:
-          version: v1.42.0
+          version: v1.44.2

.gitignore

@@ -23,7 +23,6 @@ benchmark.txt
 
 npm_licenses.tar.bz2
 /web/ui/static/react
-/web/ui/assets_vfsdata.go
 
 /vendor
 /.build

.golangci.yml

@@ -13,6 +13,7 @@ linters:
     - gofumpt
     - goimports
     - revive
+    - misspell
 
 issues:
   exclude-rules:
@@ -24,13 +25,11 @@ linters-settings:
   depguard:
     list-type: blacklist
     include-go-root: true
-    packages:
-      - sync/atomic
-      - github.com/stretchr/testify/assert
     packages-with-error-message:
       - sync/atomic: "Use go.uber.org/atomic instead of sync/atomic"
       - github.com/stretchr/testify/assert: "Use github.com/stretchr/testify/require instead of github.com/stretchr/testify/assert"
       - github.com/go-kit/kit/log: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
+      - regexp: "Use github.com/grafana/regexp instead of regexp"
   errcheck:
     exclude: scripts/errcheck_excludes.txt
   goimports:

CHANGELOG.md

@@ -1,3 +1,75 @@
+## 2.34.0 / 2022-03-15
+
+* [CHANGE] UI: Classic UI removed. #10208
+* [CHANGE] Tracing: Migrate from Jaeger to OpenTelemetry based tracing. #9724, #10203, #10276
+* [ENHANCEMENT] TSDB: Disable the chunk write queue by default and allow configuration with the experimental flag `--storage.tsdb.head-chunks-write-queue-size`. #10425
+* [ENHANCEMENT] HTTP SD: Add a failure counter. #10372
+* [ENHANCEMENT] Azure SD: Set Prometheus User-Agent on requests. #10209
+* [ENHANCEMENT] Uyuni SD: Reduce the number of logins to Uyuni. #10072
+* [ENHANCEMENT] Scrape: Log when an invalid media type is encountered during a scrape. #10186
+* [ENHANCEMENT] Scrape: Accept application/openmetrics-text;version=1.0.0 in addition to version=0.0.1. #9431
+* [ENHANCEMENT] Remote-read: Add an option to not use external labels as selectors for remote read. #10254
+* [ENHANCEMENT] UI: Optimize the alerts page and add a search bar. #10142
+* [ENHANCEMENT] UI: Improve graph colors that were hard to see. #10179
+* [ENHANCEMENT] Config: Allow escaping of `$` with `$$` when using environment variables with external labels. #10129
+* [BUGFIX] PromQL: Properly return an error from histogram_quantile when metrics have the same labelset. #10140 
+* [BUGFIX] UI: Fix bug that sets the range input to the resolution. #10227
+* [BUGFIX] TSDB: Fix a query panic when `memory-snapshot-on-shutdown` is enabled. #10348
+* [BUGFIX] Parser: Specify type in metadata parser errors. #10269
+* [BUGFIX] Scrape: Fix label limit changes not applying. #10370
+
+## 2.33.5 / 2022-03-08
+
+The binaries published with this release are built with Go1.17.8 to avoid [CVE-2022-24921](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-24921).
+
+* [BUGFIX] Remote-write: Fix deadlock between adding to queue and getting batch. #10395
+
+## 2.33.4 / 2022-02-22
+
+* [BUGFIX] TSDB: Fix panic when m-mapping head chunks onto the disk. #10316
+
+## 2.33.3 / 2022-02-11
+
+* [BUGFIX] Azure SD: Fix a regression when public IP Address isn't set. #10289
+
+## 2.33.2 / 2022-02-11
+
+* [BUGFIX] Azure SD: Fix panic when public IP Address isn't set. #10280
+* [BUGFIX] Remote-write: Fix deadlock when stopping a shard. #10279
+
+## 2.33.1 / 2022-02-02
+
+* [BUGFIX] SD: Fix _no such file or directory_ in K8s SD when not running inside K8s. #10235
+
+## 2.33.0 / 2022-01-29
+
+* [CHANGE] PromQL: Promote negative offset and `@` modifer to stable features. #10121
+* [CHANGE] Web: Promote remote-write-receiver to stable. #10119
+* [FEATURE] Config: Add `stripPort` template function. #10002
+* [FEATURE] Promtool: Add cardinality analysis to `check metrics`, enabled by flag `--extended`. #10045
+* [FEATURE] SD: Enable target discovery in own K8s namespace. #9881
+* [FEATURE] SD: Add provider ID label in K8s SD. #9603
+* [FEATURE] Web: Add limit field to the rules API. #10152
+* [ENHANCEMENT] Remote-write: Avoid allocations by buffering concrete structs instead of interfaces. #9934
+* [ENHANCEMENT] Remote-write: Log time series details for out-of-order samples in remote write receiver. #9894
+* [ENHANCEMENT] Remote-write: Shard up more when backlogged. #9274
+* [ENHANCEMENT] TSDB: Use simpler map key to improve exemplar ingest performance. #10111
+* [ENHANCEMENT] TSDB: Avoid allocations when popping from the intersected postings heap. #10092
+* [ENHANCEMENT] TSDB: Make chunk writing non-blocking, avoiding latency spikes in remote-write. #10051
+* [ENHANCEMENT] TSDB: Improve label matching performance. #9907
+* [ENHANCEMENT] UI: Optimize the service discovery page and add a search bar. #10131
+* [ENHANCEMENT] UI: Optimize the target page and add a search bar. #10103
+* [BUGFIX] Promtool: Make exit codes more consistent. #9861
+* [BUGFIX] Promtool: Fix flakiness of rule testing. #8818
+* [BUGFIX] Remote-write: Update `prometheus_remote_storage_queue_highest_sent_timestamp_seconds` metric when write irrecoverably fails. #10102
+* [BUGFIX] Storage: Avoid panic in `BufferedSeriesIterator`. #9945
+* [BUGFIX] TSDB: CompactBlockMetas should produce correct mint/maxt for overlapping blocks. #10108
+* [BUGFIX] TSDB: Fix logging of exemplar storage size. #9938
+* [BUGFIX] UI: Fix overlapping click targets for the alert state checkboxes. #10136
+* [BUGFIX] UI: Fix _Unhealthy_ filter on target page to actually display only _Unhealthy_ targets. #10103
+* [BUGFIX] UI: Fix autocompletion when expression is empty. #10053
+* [BUGFIX] TSDB: Fix deadlock from simultaneous GC and write. #10166
+
 ## 2.32.1 / 2021-12-17
 
 * [BUGFIX] Scrape: Fix reporting metrics when sample limit is reached during the report. #9996
@@ -13,7 +85,7 @@ Enable with `--enable-feature=agent`.
 
 Learn more about the Prometheus Agent in our [blog post](https://prometheus.io/blog/2021/11/16/agent/).
 
-* [CHANGE] remote-write: Change default max retry time from 100ms to 5 seconds. #9634
+* [CHANGE] Remote-write: Change default max retry time from 100ms to 5 seconds. #9634
 * [FEATURE] Agent: New mode of operation optimized for remote-write only scenarios, without local storage. Enable with `--enable-feature=agent`. #8785 #9851 #9664 #9939 #9941 #9943
 * [FEATURE] Promtool: Add `promtool check service-discovery` command. #8970
 * [FEATURE] UI: Add search in metrics dropdown. #9629
@@ -36,6 +108,10 @@ Learn more about the Prometheus Agent in our [blog post](https://prometheus.io/b
 * [BUGFIX] Uyuni SD: Fix null pointer exception during initialization. #9924 #9950
 * [BUGFIX] TSDB: Fix queries after a failed snapshot replay. #9980
 
+## 2.31.2 / 2021-12-09
+
+* [BUGFIX] TSDB: Fix queries after a failed snapshot replay. #9980
+
 ## 2.31.1 / 2021-11-05
 
 * [BUGFIX] SD: Fix a panic when the experimental discovery manager receives
@@ -66,6 +142,10 @@ Learn more about the Prometheus Agent in our [blog post](https://prometheus.io/b
 * [BUGFIX] TSDB: Fix memory leak in samples deletion. #9151
 * [BUGFIX] UI: Use consistent margin-bottom for all alert kinds. #9318
 
+## 2.30.4 / 2021-12-09
+
+* [BUGFIX] TSDB: Fix queries after a failed snapshot replay. #9980
+
 ## 2.30.3 / 2021-10-05
 
 * [BUGFIX] TSDB: Fix panic on failed snapshot replay. #9438

Makefile

@@ -43,20 +43,19 @@ ui-build-module:
 
 .PHONY: ui-test
 ui-test:
-	cd $(UI_PATH) && npm run test:coverage
+	cd $(UI_PATH) && CI=true npm run test:coverage
 
 .PHONY: ui-lint
 ui-lint:
 	cd $(UI_PATH) && npm run lint
 
 .PHONY: assets
 assets: ui-install ui-build
-	@echo ">> writing assets"
-	# Un-setting GOOS and GOARCH here because the generated Go code is always the same,
-	# but the cached object code is incompatible between architectures and OSes (which
-	# breaks cross-building for different combinations on CI in the same container).
-	cd $(UI_PATH) && GO111MODULE=$(GO111MODULE) GOOS= GOARCH= $(GO) generate -x -v $(GOOPTS)
-	@$(GOFMT) -w ./$(UI_PATH)
+
+.PHONY: assets-compress
+assets-compress:
+	@echo '>> compressing assets'
+	scripts/compress_assets.sh
 
 .PHONY: test
 # If we only want to only test go code we have to change the test target
@@ -80,7 +79,7 @@ tarball: npm_licenses common-tarball
 docker: npm_licenses common-docker
 
 .PHONY: build
-build: assets common-build
+build: assets assets-compress common-build
 
 .PHONY: bench_tsdb
 bench_tsdb: $(PROMU)

Makefile.common

@@ -83,7 +83,7 @@ PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_
 
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.42.0
+GOLANGCI_LINT_VERSION ?= v1.44.2
 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))

README.md

@@ -56,7 +56,7 @@ Prometheus will now be reachable at http://localhost:9090/.
 ### Building from source
 
 To build Prometheus from source code, You need:
-* Go [version 1.14 or greater](https://golang.org/doc/install).
+* Go [version 1.16 or greater](https://golang.org/doc/install).
 * NodeJS [version 16 or greater](https://nodejs.org/).
 * npm [version 7 or greater](https://www.npmjs.com/).