support downsampled blocks

Signed-off-by: yeya24 <yb532204897@gmail.com>
thanos-io · May 18, 2021 · f1a7888 · f1a7888
1 parent c9a3774
commit f1a7888
Show file tree

Hide file tree

Showing 5 changed files with 295 additions and 68 deletions.
diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go
@@ -268,7 +268,7 @@ func runCompact(
 				ignoreDeletionMarkFilter,
 				duplicateBlocksFilter,
 				noCompactMarkerFilter,
-			}, []block.MetadataModifier{block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels, conf.dedupFunc == compact.DedupAlgorithmPenalty)},
+			}, []block.MetadataModifier{block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels)},
 		)
 		cf.UpdateOnChange(func(blocks []metadata.Meta, err error) {
 			compactorView.Set(blocks, err)

diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go
@@ -583,7 +583,7 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat
 					block.NewConsistencyDelayMetaFilter(logger, *consistencyDelay, extprom.WrapRegistererWithPrefix(extpromPrefix, reg)),
 					ignoreDeletionMarkFilter,
 					duplicateBlocksFilter,
-				}, []block.MetadataModifier{block.NewReplicaLabelRemover(logger, make([]string, 0), false)},
+				}, []block.MetadataModifier{block.NewReplicaLabelRemover(logger, make([]string, 0))},
 			)
 			sy, err = compact.NewMetaSyncer(
 				logger,

diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go
@@ -697,12 +697,11 @@ type ReplicaLabelRemover struct {
 	logger log.Logger
 
 	replicaLabels []string
-	dedupEnabled  bool
 }
 
 // NewReplicaLabelRemover creates a ReplicaLabelRemover.
-func NewReplicaLabelRemover(logger log.Logger, replicaLabels []string, dedupEnabled bool) *ReplicaLabelRemover {
-	return &ReplicaLabelRemover{logger: logger, replicaLabels: replicaLabels, dedupEnabled: dedupEnabled}
+func NewReplicaLabelRemover(logger log.Logger, replicaLabels []string) *ReplicaLabelRemover {
+	return &ReplicaLabelRemover{logger: logger, replicaLabels: replicaLabels}
 }
 
 // Modify modifies external labels of existing blocks, it removes given replica labels from the metadata of blocks that have it.
@@ -712,11 +711,6 @@ func (r *ReplicaLabelRemover) Modify(_ context.Context, metas map[ulid.ULID]*met
 	}
 
 	for u, meta := range metas {
-		// Skip downsampled blocks for now if penalty based deduplication is enabled.
-		// TODO: remove this after downsampled blocks are supported.
-		if r.dedupEnabled && meta.Thanos.Downsample.Resolution != int64(0) {
-			continue
-		}
 		l := meta.Thanos.Labels
 		for _, replicaLabel := range r.replicaLabels {
 			if _, exists := l[replicaLabel]; exists {

diff --git a/pkg/block/fetcher_test.go b/pkg/block/fetcher_test.go
@@ -899,7 +899,7 @@ func TestReplicaLabelRemover_Modify(t *testing.T) {
 				ULID(3): {Thanos: metadata.Thanos{Labels: map[string]string{"message": "something1"}}},
 			},
 			modified:            0,
-			replicaLabelRemover: NewReplicaLabelRemover(log.NewNopLogger(), []string{"replica", "rule_replica"}, false),
+			replicaLabelRemover: NewReplicaLabelRemover(log.NewNopLogger(), []string{"replica", "rule_replica"}),
 		},
 		{
 			name: "with replica labels",
@@ -916,7 +916,7 @@ func TestReplicaLabelRemover_Modify(t *testing.T) {
 				ULID(4): {Thanos: metadata.Thanos{Labels: map[string]string{"replica": "deduped"}}},
 			},
 			modified:            5.0,
-			replicaLabelRemover: NewReplicaLabelRemover(log.NewNopLogger(), []string{"replica", "rule_replica"}, false),
+			replicaLabelRemover: NewReplicaLabelRemover(log.NewNopLogger(), []string{"replica", "rule_replica"}),
 		},
 		{
 			name: "no replica label specified in the ReplicaLabelRemover",
@@ -931,7 +931,7 @@ func TestReplicaLabelRemover_Modify(t *testing.T) {
 				ULID(3): {Thanos: metadata.Thanos{Labels: map[string]string{"message": "something1"}}},
 			},
 			modified:            0,
-			replicaLabelRemover: NewReplicaLabelRemover(log.NewNopLogger(), []string{}, false),
+			replicaLabelRemover: NewReplicaLabelRemover(log.NewNopLogger(), []string{}),
 		},
 	} {
 		m := newTestFetcherMetrics()