Merge branch 'main' into chore/bucket-upload-snapshot

thanos-io · Dec 18, 2023 · 59cfc68 · 59cfc68
2 parents b287964 + 96b8dba
commit 59cfc68
Show file tree

Hide file tree

Showing 120 changed files with 2,109 additions and 1,239 deletions.
diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk
@@ -35,12 +35,6 @@ $(FAILLINT): $(BINGO_DIR)/faillint.mod
 	@echo "(re)installing $(GOBIN)/faillint-v1.11.0"
 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=faillint.mod -o=$(GOBIN)/faillint-v1.11.0 "github.com/fatih/faillint"
 
-GO_BINDATA := $(GOBIN)/go-bindata-v3.1.1+incompatible
-$(GO_BINDATA): $(BINGO_DIR)/go-bindata.mod
-	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
-	@echo "(re)installing $(GOBIN)/go-bindata-v3.1.1+incompatible"
-	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=go-bindata.mod -o=$(GOBIN)/go-bindata-v3.1.1+incompatible "github.com/go-bindata/go-bindata/go-bindata"
-
 GOIMPORTS := $(GOBIN)/goimports-v0.12.0
 $(GOIMPORTS): $(BINGO_DIR)/goimports.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.

diff --git a/.bingo/go-bindata.mod b/.bingo/go-bindata.mod
diff --git a/.bingo/go-bindata.sum b/.bingo/go-bindata.sum
diff --git a/.bingo/variables.env b/.bingo/variables.env
@@ -14,8 +14,6 @@ BINGO="${GOBIN}/bingo-v0.8.1-0.20230820182247-0568407746a2"
 
 FAILLINT="${GOBIN}/faillint-v1.11.0"
 
-GO_BINDATA="${GOBIN}/go-bindata-v3.1.1+incompatible"
-
 GOIMPORTS="${GOBIN}/goimports-v0.12.0"
 
 GOJSONTOYAML="${GOBIN}/gojsontoyaml-v0.1.0"

diff --git a/.busybox-versions b/.busybox-versions
@@ -1,7 +1,7 @@
 # Auto generated by busybox-updater.sh. DO NOT EDIT
-amd64=9990834ed59b823ba7ea1374a4c4233f692926312ac3a1ae6b13b47be07d6934
-arm64=e96ea17e7b6a5fe22366931fe90dd5307bd6f5a3612ae14490d1b5f68b291476
-arm=a84fbdcadfd8715f11877dd3d963342e427a8718b1aa2d194115c85982cb33c5
-ppc64le=32a0ce3a915ec8b7b2ed4fb4e19a85d85af68384312fc9721b8f872ca03274a8
-riscv64=8bd68bf643b76d1b55788aee51399f6f2b910c5975dbad4ca9e624f88dd2e892
-s390x=b6863ec884fcce31cd1c7c3bb3bf6db1190ccc20845cee6dc034d82687e1b36c
+amd64=f173c44fab35484fa0e940e42929efe2a2f506feda431ba72c5f0d79639d7f55
+arm64=6277ab6abe348994989b3959d7c125d7a487012aedb80570ec28652a012c69d6
+arm=31533906c9eadc190de436bcbc021207d90839777c1b95991edd15e7df5d34ad
+ppc64le=ace2ea29bf8e4267c293ed5570df249039a0bc3949d3371429a71cf114e8a9e2
+riscv64=ff38cae5b5ed16251631e55156a7d92977bf5a8c4714ff6e3c333f7acb0297e0
+s390x=59d0ed3060aef57d1b23bc353a2223af24a6e1d035486647eb599a77ff2d446e
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,7 +2,7 @@
 version: 2.1
 
 orbs:
-  go: circleci/go@1.7.2
+  go: circleci/go@1.7.3
   git-shallow-clone: guitarrapc/git-shallow-clone@2.5.0
 
 executors:
@@ -21,7 +21,13 @@ jobs:
       GO111MODULE: "on"
     steps:
       - git-shallow-clone/checkout
-      - go/mod-download-cached
+      - go/load-cache
+      - go/mod-download
+      - run:
+          name: Download bingo modules
+          command: |
+            make install-tool-deps
+      - go/save-cache
       - setup_remote_docker:
           version: 20.10.12
       - run:

diff --git a/.dockerignore b/.dockerignore
@@ -24,7 +24,4 @@ website/public/
 website/docs-pre-processed/
 !website/data
 
-# React build assets
-pkg/ui/static/react
-
 tmp/
diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
@@ -70,8 +70,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        parallelism: [4]
-        index: [0, 1, 2, 3]
+        parallelism: [8]
+        index: [0, 1, 2, 3, 4, 5, 6, 7]
     runs-on: ubuntu-latest
     name: Thanos end-to-end tests
     env:

diff --git a/.gitignore b/.gitignore
@@ -35,9 +35,6 @@ website/public/
 website/docs-pre-processed/
 !website/data
 
-# React build assets
-pkg/ui/static/react
-
 tmp/bin
 examples/tmp/
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,13 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#6874](https://github.com/thanos-io/thanos/pull/6874) Sidecar: fix labels returned by 'api/v1/series' in presence of conflicting external and inner labels.
 
 ### Added
+- [#6944](https://github.com/thanos-io/thanos/pull/6944) Receive: Added a new flag for maximum retention bytes.
+- [#6891](https://github.com/thanos-io/thanos/pull/6891) Objstore: Bump `objstore` which adds support for Azure Workload Identity.
+- [#6453](https://github.com/thanos-io/thanos/pull/6453) Sidecar: Added `--reloader.method` to support configuration reloads via SIHUP signal.
+- [#6925](https://github.com/thanos-io/thanos/pull/6925) Store Gateway: Support float native histogram.
+- [#6954](https://github.com/thanos-io/thanos/pull/6954) Index Cache: Support tracing for fetch APIs.
+- [#6943](https://github.com/thanos-io/thanos/pull/6943) Ruler: Added `keep_firing_for` field in alerting rule.
+- [#6972](https://github.com/thanos-io/thanos/pull/6972) Store Gateway: Apply series limit when streaming series for series actually matched if lazy postings is enabled.
 
 ### Changed
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -84,7 +84,6 @@ It's key to get familiarized with the style guide and mechanics of Thanos, espec
 * To get started, create a codespace for this repository by clicking this 👉 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=109162639)
   * A codespace will open in a web-based version of Visual Studio Code. The [dev container](.devcontainer/devcontainer.json) is fully configured with software needed for this project.
   * **Note**: Dev containers is an open spec which is supported by [GitHub Codespaces](https://github.com/codespaces) and [other tools](https://containers.dev/supporting).
-* Spin up a prebuilt dev environment using Gitpod.io [![Gitpod Ready-to-Code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/thanos-io/thanos)
 * In case you want to develop the project locally, install **Golang** in your machine. Here is a nice [gist](https://gist.github.com/nikhita/432436d570b89cab172dcf2894465753) for this purpose.
 * You can run an interactive example, which populates some data as well, by following the steps mentioned [here](https://github.com/thanos-io/thanos/blob/main/tutorials/interactive-example/README.md).
 

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -1,19 +1,19 @@
 # Core Maintainers of this repository
 
-| Name                  | Email                     | Slack                    | GitHub                                           | Company             |
-|-----------------------|---------------------------|--------------------------|--------------------------------------------------|---------------------|
-| Bartłomiej Płotka     | bwplotka@gmail.com        | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)         | Red Hat             |
-| Frederic Branczyk     | fbranczyk@gmail.com       | `@brancz`                | [@brancz](https://github.com/brancz)             | Polar Signals       |
-| Giedrius Statkevičius | giedriuswork@gmail.com    | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)       | Vinted              |
-| Kemal Akkoyun         | kakkoyun@gmail.com        | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)         | Polar Signals       |
-| Lucas Servén Marín    | lserven@gmail.com         | `@squat`                 | [@squat](https://github.com/squat)               | Red Hat             |
-| Prem Saraswat         | prmsrswt@gmail.com        | `@Prem Saraswat`         | [@onprem](https://github.com/onprem)             | Red Hat             |
-| Matthias Loibl        | mail@matthiasloibl.com    | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)     | Polar Signals       |
-| Ben Ye                | yb532204897@gmail.com     | `@yeya24`                | [@yeya24](https://github.com/yeya24)             | Amazon Web Services |
-| Wiard van Rij         | wiard@outlook.com         | `@wiard van Rij`         | [@wiardvanrij](https://github.com/wiardvanrij)   | Roku                |
-| Matej Gera            | matejgera@gmail.com       | `@Matej Gera`            | [@matej-g](https://github.com/matej-g)           | Red Hat             |
-| Filip Petkovski       | filip.petkovsky@gmail.com | `@Filip Petkovski`       | [@fpetkovski](https://github.com/fpetkovski)     | Shopify             |
-| Saswata Mukherjee     | saswata.mukhe@gmail.com   | `@saswatamcode`          | [@saswatamcode](https://github.com/saswatamcode) | Red Hat             |
+| Name                  | Email                     | Slack                    | GitHub                                             | Company             |
+|-----------------------|---------------------------|--------------------------|----------------------------------------------------|---------------------|
+| Bartłomiej Płotka     | bwplotka@gmail.com        | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)           | Red Hat             |
+| Frederic Branczyk     | fbranczyk@gmail.com       | `@brancz`                | [@brancz](https://github.com/brancz)               | Polar Signals       |
+| Giedrius Statkevičius | giedriuswork@gmail.com    | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)         | Vinted              |
+| Kemal Akkoyun         | kakkoyun@gmail.com        | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)           | Polar Signals       |
+| Lucas Servén Marín    | lserven@gmail.com         | `@squat`                 | [@squat](https://github.com/squat)                 | Red Hat             |
+| Prem Saraswat         | prmsrswt@gmail.com        | `@Prem Saraswat`         | [@onprem](https://github.com/onprem)               | Red Hat             |
+| Matthias Loibl        | mail@matthiasloibl.com    | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)       | Polar Signals       |
+| Ben Ye                | yb532204897@gmail.com     | `@yeya24`                | [@yeya24](https://github.com/yeya24)               | Amazon Web Services |
+| Matej Gera            | matejgera@gmail.com       | `@Matej Gera`            | [@matej-g](https://github.com/matej-g)             | Coralogix           |
+| Filip Petkovski       | filip.petkovsky@gmail.com | `@Filip Petkovski`       | [@fpetkovski](https://github.com/fpetkovski)       | Shopify             |
+| Saswata Mukherjee     | saswata.mukhe@gmail.com   | `@saswatamcode`          | [@saswatamcode](https://github.com/saswatamcode)   | Red Hat             |
+| Michael Hoffmann      | mhoffm@posteo.de          | `@Michael Hoffmann`      | [@MichaHoffmann](https://github.com/MichaHoffmann) | Aiven               |
 
 We are bunch of people from different companies with various interests and skills. We are from different parts of the world: Germany, Holland, Lithuania, US, UK and India. We have something in common though: We all share the love for OpenSource, Go, Prometheus, :coffee: and Observability topics.
 
@@ -31,16 +31,15 @@ We also have some nice souls that help triaging issues and PRs. See [here](https
 
 Full list of triage persons is displayed below:
 
-| Name             | Slack               | GitHub                                             | Company |
-|------------------|---------------------|----------------------------------------------------|---------|
-| Adrien Fillon    | `@Adrien F`         | [@adrien-f](https://github.com/adrien-f)           |         |
-| Ian Billett      | `@billett`          | [@bill3tt](https://github.com/bill3tt)             | Red Hat |
-| Martin Chodur    | `@FUSAKLA`          | [@fusakla](https://github.com/fusakla)             |         |
-| Michael Dai      | `@jojohappy`        | [@jojohappy](https://github.com/jojohappy)         |         |
-| Xiang Dai        | `@daixiang0`        | [@daixiang0](https://github.com/daixiang0)         |         |
-| Jimmie Han       | `@hanjm`            | [@hanjm](https://github.com/hanjm)                 | Tencent |
-| Michael Hoffmann | `@Michael Hoffmann` | [@MichaHoffmann](https://github.com/MichaHoffmann) | Aiven   |
-| Douglas Camata   | `@douglascamata`    | [@douglascamata](https://github.com/douglascamata) | Red Hat |
+| Name           | Slack            | GitHub                                             | Company |
+|----------------|------------------|----------------------------------------------------|---------|
+| Adrien Fillon  | `@Adrien F`      | [@adrien-f](https://github.com/adrien-f)           |         |
+| Ian Billett    | `@billett`       | [@bill3tt](https://github.com/bill3tt)             | Red Hat |
+| Martin Chodur  | `@FUSAKLA`       | [@fusakla](https://github.com/fusakla)             |         |
+| Michael Dai    | `@jojohappy`     | [@jojohappy](https://github.com/jojohappy)         |         |
+| Xiang Dai      | `@daixiang0`     | [@daixiang0](https://github.com/daixiang0)         |         |
+| Jimmie Han     | `@hanjm`         | [@hanjm](https://github.com/hanjm)                 | Tencent |
+| Douglas Camata | `@douglascamata` | [@douglascamata](https://github.com/douglascamata) | Red Hat |
 
 Please reach any of the maintainer on slack or email if you want to help as well.
 

diff --git a/Makefile b/Makefile
@@ -118,14 +118,8 @@ $(REACT_APP_OUTPUT_DIR): $(REACT_APP_NODE_MODULES_PATH) $(REACT_APP_SOURCE_FILES
 	   @echo ">> building React app"
 	   @scripts/build-react-app.sh
 
-.PHONY: assets
-assets: # Repacks all static assets into go file for easier deploy.
-assets: $(GO_BINDATA) $(REACT_APP_OUTPUT_DIR)
-	@echo ">> deleting asset file"
-	@rm pkg/ui/bindata.go || true
-	@echo ">> writing assets"
-	@$(GO_BINDATA) $(bindata_flags) -pkg ui -o pkg/ui/bindata.go  pkg/ui/static/...
-	@$(MAKE) format
+.PHONY: react-app
+react-app: $(REACT_APP_OUTPUT_DIR)
 
 .PHONY: react-app-lint
 react-app-lint: $(REACT_APP_NODE_MODULES_PATH)

diff --git a/README.md b/README.md
@@ -4,6 +4,8 @@
 
 [![CI](https://github.com/thanos-io/thanos/workflows/CI/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3ACI) [![CI](https://circleci.com/gh/thanos-io/thanos.svg?style=svg)](https://circleci.com/gh/thanos-io/thanos) [![go](https://github.com/thanos-io/thanos/workflows/go/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Ago) [![react](https://github.com/thanos-io/thanos/workflows/react/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Areact) [![docs](https://github.com/thanos-io/thanos/workflows/docs/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Adocs) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/thanos-io/thanos) [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=109162639)
 
+> 📢 [ThanosCon](https://thanos.io/blog/2023-20-11-thanoscon/) is happening on 19th March as a co-located half-day on KubeCon EU in Paris. Join us there! 🤗 CFP is open until 3rd December!
+
 ## Overview
 
 Thanos is a set of components that can be composed into a highly available metric system with unlimited storage capacity, which can be added seamlessly on top of existing Prometheus deployments.

diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go
@@ -234,11 +234,13 @@ func runCompact(
 	ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, insBkt, deleteDelay/2, conf.blockMetaFetchConcurrency)
 	duplicateBlocksFilter := block.NewDeduplicateFilter(conf.blockMetaFetchConcurrency)
 	noCompactMarkerFilter := compact.NewGatherNoCompactionMarkFilter(logger, insBkt, conf.blockMetaFetchConcurrency)
+	noDownsampleMarkerFilter := downsample.NewGatherNoDownsampleMarkFilter(logger, insBkt, conf.blockMetaFetchConcurrency)
 	labelShardedMetaFilter := block.NewLabelShardedMetaFilter(relabelConfig)
 	consistencyDelayMetaFilter := block.NewConsistencyDelayMetaFilter(logger, conf.consistencyDelay, extprom.WrapRegistererWithPrefix("thanos_", reg))
 	timePartitionMetaFilter := block.NewTimePartitionMetaFilter(conf.filterConf.MinTime, conf.filterConf.MaxTime)
 
-	baseMetaFetcher, err := block.NewBaseFetcher(logger, conf.blockMetaFetchConcurrency, insBkt, conf.dataDir, extprom.WrapRegistererWithPrefix("thanos_", reg))
+	baseBlockIDsFetcher := block.NewBaseBlockIDsFetcher(logger, insBkt)
+	baseMetaFetcher, err := block.NewBaseFetcher(logger, conf.blockMetaFetchConcurrency, insBkt, baseBlockIDsFetcher, conf.dataDir, extprom.WrapRegistererWithPrefix("thanos_", reg))
 	if err != nil {
 		return errors.Wrap(err, "create meta fetcher")
 	}
@@ -260,18 +262,21 @@ func runCompact(
 		sy  *compact.Syncer
 	)
 	{
+		filters := []block.MetadataFilter{
+			timePartitionMetaFilter,
+			labelShardedMetaFilter,
+			consistencyDelayMetaFilter,
+			ignoreDeletionMarkFilter,
+			block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels),
+			duplicateBlocksFilter,
+			noCompactMarkerFilter,
+		}
+		if !conf.disableDownsampling {
+			filters = append(filters, noDownsampleMarkerFilter)
+		}
 		// Make sure all compactor meta syncs are done through Syncer.SyncMeta for readability.
 		cf := baseMetaFetcher.NewMetaFetcher(
-			extprom.WrapRegistererWithPrefix("thanos_", reg), []block.MetadataFilter{
-				timePartitionMetaFilter,
-				labelShardedMetaFilter,
-				consistencyDelayMetaFilter,
-				ignoreDeletionMarkFilter,
-				block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels),
-				duplicateBlocksFilter,
-				noCompactMarkerFilter,
-			},
-		)
+			extprom.WrapRegistererWithPrefix("thanos_", reg), filters)
 		cf.UpdateOnChange(func(blocks []metadata.Meta, err error) {
 			api.SetLoaded(blocks, err)
 		})
@@ -436,22 +441,53 @@ func runCompact(
 				return errors.Wrap(err, "sync before first pass of downsampling")
 			}
 
-			for _, meta := range sy.Metas() {
+			filteredMetas := sy.Metas()
+			noDownsampleBlocks := noDownsampleMarkerFilter.NoDownsampleMarkedBlocks()
+			for ul := range noDownsampleBlocks {
+				delete(filteredMetas, ul)
+			}
+
+			for _, meta := range filteredMetas {
 				groupKey := meta.Thanos.GroupKey()
 				downsampleMetrics.downsamples.WithLabelValues(groupKey)
 				downsampleMetrics.downsampleFailures.WithLabelValues(groupKey)
 			}
-			if err := downsampleBucket(ctx, logger, downsampleMetrics, insBkt, sy.Metas(), downsamplingDir, conf.downsampleConcurrency, conf.blockFilesConcurrency, metadata.HashFunc(conf.hashFunc), conf.acceptMalformedIndex); err != nil {
+
+			if err := downsampleBucket(
+				ctx,
+				logger,
+				downsampleMetrics,
+				insBkt,
+				filteredMetas,
+				downsamplingDir,
+				conf.downsampleConcurrency,
+				conf.blockFilesConcurrency,
+				metadata.HashFunc(conf.hashFunc),
+				conf.acceptMalformedIndex,
+			); err != nil {
 				return errors.Wrap(err, "first pass of downsampling failed")
 			}
 
 			level.Info(logger).Log("msg", "start second pass of downsampling")
 			if err := sy.SyncMetas(ctx); err != nil {
 				return errors.Wrap(err, "sync before second pass of downsampling")
 			}
-			if err := downsampleBucket(ctx, logger, downsampleMetrics, insBkt, sy.Metas(), downsamplingDir, conf.downsampleConcurrency, conf.blockFilesConcurrency, metadata.HashFunc(conf.hashFunc), conf.acceptMalformedIndex); err != nil {
+
+			if err := downsampleBucket(
+				ctx,
+				logger,
+				downsampleMetrics,
+				insBkt,
+				filteredMetas,
+				downsamplingDir,
+				conf.downsampleConcurrency,
+				conf.blockFilesConcurrency,
+				metadata.HashFunc(conf.hashFunc),
+				conf.acceptMalformedIndex,
+			); err != nil {
 				return errors.Wrap(err, "second pass of downsampling failed")
 			}
+
 			level.Info(logger).Log("msg", "downsampling iterations done")
 		} else {
 			level.Info(logger).Log("msg", "downsampling was explicitly disabled")