Added block index attributes caching support to metadata cache

CHANGELOG.md

@@ -18,6 +18,7 @@
 * [ENHANCEMENT] Ruler: Add `cortex_prometheus_last_evaluation_samples` to expose the number of samples generated by a rule group per tenant. #3582
 * [ENHANCEMENT] Memberlist: add status page (/memberlist) with available details about memberlist-based KV store and memberlist cluster. It's also possible to view KV values in Go struct or JSON format, or download for inspection. #3575
 * [ENHANCEMENT] Memberlist: client can now keep a size-bounded buffer with sent and received messages and display them in the admin UI (/memberlist) for troubleshooting. #3581 #3602
+* [ENHANCEMENT] Blocks storage: added block index attributes caching support to metadata cache. The TTL can be configured via `-blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl`. #3629
 * [BUGFIX] Allow `-querier.max-query-lookback` use `y|w|d` suffix like deprecated `-store.max-look-back-period`. #3598
 * [BUGFIX] Query-Frontend: `cortex_query_seconds_total` now return seconds not nanoseconds. #3589
 * [BUGFIX] Memberlist: Entry in the ring should now not appear again after using "Forget" feature (unless it's still heartbeating). #3603

docs/blocks-storage/querier.md

@@ -608,6 +608,10 @@ blocks_storage:
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl
       [metafile_attributes_ttl: <duration> | default = 168h]
 
+      # How long to cache attributes of the block index.
+      # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl
+      [block_index_attributes_ttl: <duration> | default = 168h]
+
       # How long to cache content of the bucket index.
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl
       [bucket_index_content_ttl: <duration> | default = 5m]

docs/blocks-storage/store-gateway.md

@@ -642,6 +642,10 @@ blocks_storage:
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl
       [metafile_attributes_ttl: <duration> | default = 168h]
 
+      # How long to cache attributes of the block index.
+      # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl
+      [block_index_attributes_ttl: <duration> | default = 168h]
+
       # How long to cache content of the bucket index.
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl
       [bucket_index_content_ttl: <duration> | default = 5m]

docs/configuration/config-file-reference.md

@@ -3870,6 +3870,10 @@ bucket_store:
     # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl
     [metafile_attributes_ttl: <duration> | default = 168h]
 
+    # How long to cache attributes of the block index.
+    # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl
+    [block_index_attributes_ttl: <duration> | default = 168h]
+
     # How long to cache content of the bucket index.
     # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl
     [bucket_index_content_ttl: <duration> | default = 5m]

pkg/storage/tsdb/caching_bucket.go

@@ -3,14 +3,17 @@ package tsdb
 import (
 	"flag"
 	"fmt"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
 
 	"github.com/go-kit/kit/log"
 	"github.com/golang/snappy"
+	"github.com/oklog/ulid"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/cache"
 	"github.com/thanos-io/thanos/pkg/cacheutil"
@@ -69,16 +72,17 @@ func (cfg *ChunksCacheConfig) Validate() error {
 type MetadataCacheConfig struct {
 	CacheBackend `yaml:",inline"`
 
-	TenantsListTTL         time.Duration `yaml:"tenants_list_ttl"`
-	TenantBlocksListTTL    time.Duration `yaml:"tenant_blocks_list_ttl"`
-	ChunksListTTL          time.Duration `yaml:"chunks_list_ttl"`
-	MetafileExistsTTL      time.Duration `yaml:"metafile_exists_ttl"`
-	MetafileDoesntExistTTL time.Duration `yaml:"metafile_doesnt_exist_ttl"`
-	MetafileContentTTL     time.Duration `yaml:"metafile_content_ttl"`
-	MetafileMaxSize        int           `yaml:"metafile_max_size_bytes"`
-	MetafileAttributesTTL  time.Duration `yaml:"metafile_attributes_ttl"`
-	BucketIndexContentTTL  time.Duration `yaml:"bucket_index_content_ttl"`
-	BucketIndexMaxSize     int           `yaml:"bucket_index_max_size_bytes"`
+	TenantsListTTL          time.Duration `yaml:"tenants_list_ttl"`
+	TenantBlocksListTTL     time.Duration `yaml:"tenant_blocks_list_ttl"`
+	ChunksListTTL           time.Duration `yaml:"chunks_list_ttl"`
+	MetafileExistsTTL       time.Duration `yaml:"metafile_exists_ttl"`
+	MetafileDoesntExistTTL  time.Duration `yaml:"metafile_doesnt_exist_ttl"`
+	MetafileContentTTL      time.Duration `yaml:"metafile_content_ttl"`
+	MetafileMaxSize         int           `yaml:"metafile_max_size_bytes"`
+	MetafileAttributesTTL   time.Duration `yaml:"metafile_attributes_ttl"`
+	BlockIndexAttributesTTL time.Duration `yaml:"block_index_attributes_ttl"`
+	BucketIndexContentTTL   time.Duration `yaml:"bucket_index_content_ttl"`
+	BucketIndexMaxSize      int           `yaml:"bucket_index_max_size_bytes"`
 }
 
 func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
@@ -94,6 +98,7 @@ func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix
 	f.DurationVar(&cfg.MetafileContentTTL, prefix+"metafile-content-ttl", 24*time.Hour, "How long to cache content of the metafile.")
 	f.IntVar(&cfg.MetafileMaxSize, prefix+"metafile-max-size-bytes", 1*1024*1024, "Maximum size of metafile content to cache in bytes. Caching will be skipped if the content exceeds this size. This is useful to avoid network round trip for large content if the configured caching backend has an hard limit on cached items size (in this case, you should set this limit to the same limit in the caching backend).")
 	f.DurationVar(&cfg.MetafileAttributesTTL, prefix+"metafile-attributes-ttl", 168*time.Hour, "How long to cache attributes of the block metafile.")
+	f.DurationVar(&cfg.BlockIndexAttributesTTL, prefix+"block-index-attributes-ttl", 168*time.Hour, "How long to cache attributes of the block index.")
 	f.DurationVar(&cfg.BucketIndexContentTTL, prefix+"bucket-index-content-ttl", 5*time.Minute, "How long to cache content of the bucket index.")
 	f.IntVar(&cfg.BucketIndexMaxSize, prefix+"bucket-index-max-size-bytes", 1*1024*1024, "Maximum size of bucket index content to cache in bytes. Caching will be skipped if the content exceeds this size. This is useful to avoid network round trip for large content if the configured caching backend has an hard limit on cached items size (in this case, you should set this limit to the same limit in the caching backend).")
 }
@@ -127,6 +132,7 @@ func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig Metadata
 		cfg.CacheExists("metafile", metadataCache, isMetaFile, metadataConfig.MetafileExistsTTL, metadataConfig.MetafileDoesntExistTTL)
 		cfg.CacheGet("metafile", metadataCache, isMetaFile, metadataConfig.MetafileMaxSize, metadataConfig.MetafileContentTTL, metadataConfig.MetafileExistsTTL, metadataConfig.MetafileDoesntExistTTL)
 		cfg.CacheAttributes("metafile", metadataCache, isMetaFile, metadataConfig.MetafileAttributesTTL)
+		cfg.CacheAttributes("block-index", metadataCache, isBlockIndexFile, metadataConfig.BlockIndexAttributesTTL)
 		cfg.CacheGet("bucket-index", metadataCache, isBucketIndexFile, metadataConfig.BucketIndexMaxSize, metadataConfig.BucketIndexContentTTL /* do not cache exist / not exist: */, 0, 0)
 
 		codec := snappyIterCodec{storecache.JSONIterCodec{}}
@@ -170,6 +176,16 @@ func isMetaFile(name string) bool {
 	return strings.HasSuffix(name, "/"+metadata.MetaFilename) || strings.HasSuffix(name, "/"+metadata.DeletionMarkFilename) || strings.HasSuffix(name, "/"+TenantDeletionMarkPath)
 }
 
+func isBlockIndexFile(name string) bool {
+	// Ensure the path ends with "<block id>/<index filename>".
+	if !strings.HasSuffix(name, "/"+block.IndexFilename) {
+		return false
+	}
+
+	_, err := ulid.Parse(filepath.Base(filepath.Dir(name)))
+	return err == nil
+}
+
 func isBucketIndexFile(name string) bool {
 	// TODO can't reference bucketindex because of a circular dependency. To be fixed.
 	return strings.HasSuffix(name, "/bucket-index.json.gz")

pkg/storage/tsdb/caching_bucket_test.go

@@ -1,8 +1,10 @@
 package tsdb
 
 import (
+	"fmt"
 	"testing"
 
+	"github.com/oklog/ulid"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -21,3 +23,14 @@ func TestIsBucketIndexFile(t *testing.T) {
 	assert.False(t, isBucketIndexFile("test/block/chunks"))
 	assert.True(t, isBucketIndexFile("test/bucket-index.json.gz"))
 }
+
+func TestIsBlockIndexFile(t *testing.T) {
+	blockID := ulid.MustNew(1, nil)
+
+	assert.False(t, isBlockIndexFile(""))
+	assert.False(t, isBlockIndexFile("/index"))
+	assert.False(t, isBlockIndexFile("test/index"))
+	assert.False(t, isBlockIndexFile("/test/index"))
+	assert.True(t, isBlockIndexFile(fmt.Sprintf("%s/index", blockID.String())))
+	assert.True(t, isBlockIndexFile(fmt.Sprintf("/%s/index", blockID.String())))
+}