Adds benchmark to compare the new unique algorithm

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

Author: cyriltovena

pkg/chunk/chunk_store.go

@@ -204,17 +204,15 @@ func (c *store) LabelValuesForMetricName(ctx context.Context, userID string, fro
 		return nil, err
 	}
 
-	var result []string
-	values := map[string]struct{}{}
+	var result UniqueStrings
 	for _, entry := range entries {
 		_, labelValue, _, _, err := parseChunkTimeRangeValue(entry.RangeValue, entry.Value)
 		if err != nil {
 			return nil, err
 		}
-		result = appendUniqueStrings(result, values, string(labelValue))
+		result.Add(string(labelValue))
 	}
-	sort.Strings(result)
-	return result, nil
+	return result.Strings(), nil
 }
 
 // LabelNamesForMetricName retrieves all label names for a metric name.
@@ -462,7 +460,6 @@ func (c *store) lookupEntriesByQueries(ctx context.Context, queries []IndexQuery
 
 func (c *store) parseIndexEntries(ctx context.Context, entries []IndexEntry, matcher *labels.Matcher) ([]string, error) {
 	result := make([]string, 0, len(entries))
-	values := map[string]struct{}{}
 	for _, entry := range entries {
 		chunkKey, labelValue, _, _, err := parseChunkTimeRangeValue(entry.RangeValue, entry.Value)
 		if err != nil {
@@ -472,10 +469,11 @@ func (c *store) parseIndexEntries(ctx context.Context, entries []IndexEntry, mat
 		if matcher != nil && !matcher.Matches(string(labelValue)) {
 			continue
 		}
-		result = appendUniqueStrings(result, values, chunkKey)
+		result = append(result, chunkKey)
 	}
 	// Return ids sorted and deduped because they will be merged with other sets.
 	sort.Strings(result)
+	result = uniqueStrings(result)
 	return result, nil
 }
 

pkg/chunk/chunk_store_test.go

@@ -851,3 +851,49 @@ func TestStoreMaxLookBack(t *testing.T) {
 	require.Equal(t, 1, len(chunks))
 	chunks[0].Through.Equal(now)
 }
+
+func benchmarkParseIndexEntries(i int64, b *testing.B) {
+	b.ReportAllocs()
+	b.StopTimer()
+	store := &store{}
+	ctx := context.Background()
+	entries := generateIndexEntries(i)
+	matcher, err := labels.NewMatcher(labels.MatchRegexp, "", ".*")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.StartTimer()
+	for n := 0; n < b.N; n++ {
+		keys, err := store.parseIndexEntries(ctx, entries, matcher)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if len(keys) != len(entries)/2 {
+			b.Fatalf("expected keys:%d got:%d", len(entries)/2, len(keys))
+		}
+	}
+}
+
+func BenchmarkParseIndexEntries500(b *testing.B)   { benchmarkParseIndexEntries(500, b) }
+func BenchmarkParseIndexEntries2500(b *testing.B)  { benchmarkParseIndexEntries(2500, b) }
+func BenchmarkParseIndexEntries10000(b *testing.B) { benchmarkParseIndexEntries(10000, b) }
+func BenchmarkParseIndexEntries50000(b *testing.B) { benchmarkParseIndexEntries(50000, b) }
+
+func generateIndexEntries(n int64) []IndexEntry {
+	res := make([]IndexEntry, 0, n)
+	for i := int64(n - 1); i >= 0; i-- {
+		labelValue := fmt.Sprintf("labelvalue%d", i%(n/2))
+		chunkID := fmt.Sprintf("chunkid%d", i%(n/2))
+		rangeValue := []byte{}
+		rangeValue = append(rangeValue, []byte("component1")...)
+		rangeValue = append(rangeValue, 0)
+		rangeValue = append(rangeValue, []byte(labelValue)...)
+		rangeValue = append(rangeValue, 0)
+		rangeValue = append(rangeValue, []byte(chunkID)...)
+		rangeValue = append(rangeValue, 0)
+		res = append(res, IndexEntry{
+			RangeValue: rangeValue,
+		})
+	}
+	return res
+}

pkg/chunk/chunk_store_utils.go

@@ -2,7 +2,6 @@ package chunk
 
 import (
 	"context"
-	"sort"
 	"sync"
 
 	"github.com/go-kit/kit/log/level"
@@ -38,15 +37,13 @@ func keysFromChunks(chunks []Chunk) []string {
 }
 
 func labelNamesFromChunks(chunks []Chunk) []string {
-	keys := map[string]struct{}{}
-	var result []string
+	var result UniqueStrings
 	for _, c := range chunks {
 		for _, l := range c.Metric {
-			result = appendUniqueStrings(result, keys, string(l.Name))
+			result.Add(string(l.Name))
 		}
 	}
-	sort.Strings(result)
-	return result
+	return result.Strings()
 }
 
 func filterChunksByUniqueFingerprint(chunks []Chunk) ([]Chunk, []string) {

pkg/chunk/composite_store.go

@@ -100,34 +100,30 @@ func (c compositeStore) Get(ctx context.Context, userID string, from, through mo
 
 // LabelValuesForMetricName retrieves all label values for a single label name and metric name.
 func (c compositeStore) LabelValuesForMetricName(ctx context.Context, userID string, from, through model.Time, metricName string, labelName string) ([]string, error) {
-	var result []string
-	uniqueValues := map[string]struct{}{}
+	var result UniqueStrings
 	err := c.forStores(from, through, func(from, through model.Time, store Store) error {
 		labelValues, err := store.LabelValuesForMetricName(ctx, userID, from, through, metricName, labelName)
 		if err != nil {
 			return err
 		}
-		result = appendUniqueStrings(result, uniqueValues, labelValues...)
+		result.Add(labelValues...)
 		return nil
 	})
-	sort.Strings(result)
-	return result, err
+	return result.Strings(), err
 }
 
 // LabelNamesForMetricName retrieves all label names for a metric name.
 func (c compositeStore) LabelNamesForMetricName(ctx context.Context, userID string, from, through model.Time, metricName string) ([]string, error) {
-	var result []string
-	uniqueValues := map[string]struct{}{}
+	var result UniqueStrings
 	err := c.forStores(from, through, func(from, through model.Time, store Store) error {
 		labelNames, err := store.LabelNamesForMetricName(ctx, userID, from, through, metricName)
 		if err != nil {
 			return err
 		}
-		result = appendUniqueStrings(result, uniqueValues, labelNames...)
+		result.Add(labelNames...)
 		return nil
 	})
-	sort.Strings(result)
-	return result, err
+	return result.Strings(), err
 }
 
 func (c compositeStore) GetChunkRefs(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) ([][]Chunk, []*Fetcher, error) {

pkg/chunk/series_store.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"sort"
 
 	"github.com/go-kit/kit/log/level"
 	jsoniter "github.com/json-iterator/go"
@@ -406,18 +405,18 @@ func (c *seriesStore) lookupLabelNamesBySeries(ctx context.Context, from, throug
 		return nil, err
 	}
 	level.Debug(log).Log("entries", len(entries))
-	result := []string{model.MetricNameLabel}
-	uniqueLabelNames := map[string]struct{}{model.MetricNameLabel: {}}
+
+	var result UniqueStrings
+	result.Add(model.MetricNameLabel)
 	for _, entry := range entries {
 		lbs := []string{}
 		err := jsoniter.ConfigFastest.Unmarshal(entry.Value, &lbs)
 		if err != nil {
 			return nil, err
 		}
-		result = appendUniqueStrings(result, uniqueLabelNames, lbs...)
+		result.Add(lbs...)
 	}
-	sort.Strings(result)
-	return result, nil
+	return result.Strings(), nil
 }
 
 // Put implements ChunkStore

pkg/chunk/strings.go

@@ -1,5 +1,27 @@
 package chunk
 
+import "sort"
+
+func uniqueStrings(cs []string) []string {
+	if len(cs) == 0 {
+		return []string{}
+	}
+
+	result := make([]string, 1, len(cs))
+	result[0] = cs[0]
+	i, j := 0, 1
+	for j < len(cs) {
+		if result[i] == cs[j] {
+			j++
+			continue
+		}
+		result = append(result, cs[j])
+		i++
+		j++
+	}
+	return result
+}
+
 func intersectStrings(left, right []string) []string {
 	var (
 		i, j   = 0, 0
@@ -38,12 +60,28 @@ func nWayIntersectStrings(sets [][]string) []string {
 	}
 }
 
-func appendUniqueStrings(result []string, uniqueValues map[string]struct{}, values ...string) []string {
-	for _, l := range values {
-		if _, ok := uniqueValues[l]; !ok {
-			uniqueValues[l] = struct{}{}
-			result = append(result, l)
+// UniqueStrings keeps a slice of unique strings.
+type UniqueStrings struct {
+	values map[string]struct{}
+	result []string
+}
+
+// Add adds a new string, dropping duplicates.
+func (us *UniqueStrings) Add(strings ...string) {
+	for _, s := range strings {
+		if _, ok := us.values[s]; ok {
+			continue
 		}
+		if us.values == nil {
+			us.values = map[string]struct{}{}
+		}
+		us.values[s] = struct{}{}
+		us.result = append(us.result, s)
 	}
-	return result
+}
+
+// Strings returns the sorted sliced of unique strings.
+func (us UniqueStrings) Strings() []string {
+	sort.Strings(us.result)
+	return us.result
 }