Cherry-pick #14421 to 7.x: Improve hash performance on Prometheus collector

NOTICE.txt

@@ -338,6 +338,35 @@ License type (autodetected): Apache-2.0
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: github.com/cespare/xxhash
+Revision: 2372543dd2bbab7a55ce14c54eb357abe23a42f5
+License type (autodetected): MIT
+./vendor/github.com/cespare/xxhash/LICENSE.txt:
+--------------------------------------------------------------------
+Copyright (c) 2016 Caleb Spare
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 --------------------------------------------------------------------
 Dependency: github.com/cloudflare/cfssl
 Revision: b1ec8c586c2aa3ec3eaf4a622933f169cfa5648b
@@ -4291,15 +4320,6 @@ License type (autodetected): Apache-2.0
 Apache License 2.0
 
 
---------------------------------------------------------------------
-Dependency: github.com/OneOfOne/xxhash
-Revision: 74ace4fe5525ef62ce28d5093d6b0faaa6a575f3
-License type (autodetected): Apache-2.0
-./vendor/github.com/OneOfOne/xxhash/LICENSE:
---------------------------------------------------------------------
-Apache License 2.0
-
-
 --------------------------------------------------------------------
 Dependency: github.com/opencontainers/go-digest
 Revision: eaa60544f31ccf3b0653b1a118b76d33418ff41b

auditbeat/helper/hasher/hasher.go

@@ -30,7 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/OneOfOne/xxhash"
+	"github.com/cespare/xxhash"
 	"github.com/dustin/go-humanize"
 	"github.com/joeshaw/multierror"
 	"github.com/pkg/errors"
@@ -82,7 +82,7 @@ var validHashes = map[HashType](func() hash.Hash){
 	SHA3_384:   sha3.New384,
 	SHA3_512:   sha3.New512,
 	XXH64: func() hash.Hash {
-		return xxhash.New64()
+		return xxhash.New()
 	},
 }
 

auditbeat/module/file_integrity/event.go

@@ -33,7 +33,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/OneOfOne/xxhash"
+	"github.com/cespare/xxhash"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/blake2b"
 	"golang.org/x/crypto/sha3"
@@ -396,7 +396,7 @@ func hashFile(name string, hashType ...HashType) (map[HashType]Digest, error) {
 		case SHA512_256:
 			hashes = append(hashes, sha512.New512_256())
 		case XXH64:
-			hashes = append(hashes, xxhash.New64())
+			hashes = append(hashes, xxhash.New())
 		default:
 			return nil, errors.Errorf("unknown hash type '%v'", name)
 		}

metricbeat/helper/http.go

@@ -86,6 +86,7 @@ func newHTTPFromConfig(config Config, name string, hostData mb.HostData) (*HTTP,
 			Transport: &http.Transport{
 				Dial:    dialer.Dial,
 				DialTLS: tlsDialer.Dial,
+				Proxy:   http.ProxyFromEnvironment,
 			},
 			Timeout: config.Timeout,
 		},

metricbeat/helper/prometheus/hash.go

@@ -0,0 +1,73 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package prometheus
+
+import (
+	"bytes"
+	"sort"
+	"strconv"
+	"sync"
+
+	"github.com/cespare/xxhash"
+
+	"github.com/elastic/beats/libbeat/common"
+)
+
+const sep = '\xff'
+
+var byteBuffer = sync.Pool{
+	New: func() interface{} {
+		return bytes.NewBuffer(nil)
+	},
+}
+
+type label struct {
+	key   string
+	value string
+}
+
+type labels []label
+
+func (ls labels) Len() int           { return len(ls) }
+func (ls labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
+func (ls labels) Less(i, j int) bool { return ls[i].key < ls[j].key }
+
+// LabelHash hashes the labels map and returns a string
+func LabelHash(labelMap common.MapStr) string {
+	ls := make(labels, len(labelMap))
+
+	for k, v := range labelMap {
+		if val, ok := v.(string); ok {
+			ls = append(ls, label{k, val})
+		}
+	}
+
+	sort.Sort(ls)
+	b := byteBuffer.Get().(*bytes.Buffer)
+	b.Reset()
+
+	for _, label := range ls {
+		b.WriteString(label.key)
+		b.WriteByte(sep)
+		b.WriteString(label.value)
+		b.WriteByte(sep)
+	}
+	hash := xxhash.Sum64(b.Bytes())
+	byteBuffer.Put(b)
+	return strconv.FormatUint(hash, 10)
+}

metricbeat/helper/prometheus/hash_benchmark_test.go

@@ -0,0 +1,63 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package prometheus
+
+import (
+	"testing"
+
+	"github.com/elastic/beats/libbeat/common"
+)
+
+var testLabels = common.MapStr{
+	"request": common.MapStr{
+		"component": "apiserver",
+		"resource":  "configmaps",
+		"scope":     "cluster",
+		"verb":      "WATCH",
+		"version":   "v1",
+	},
+}
+
+var testLabelsFlat = common.MapStr{
+	"component": "apiserver",
+	"resource":  "configmaps",
+	"scope":     "cluster",
+	"verb":      "WATCH",
+	"version":   "v1",
+}
+
+func BenchmarkLabelHashWithFlatten(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		_ = LabelHash(testLabels)
+	}
+}
+
+func BenchmarkLabelHashWithOutFlatten(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		_ = LabelHash(testLabelsFlat)
+	}
+}
+
+func BenchmarkMapStrString(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		_ = testLabels.String()
+	}
+}

metricbeat/helper/prometheus/prometheus.go

@@ -230,7 +230,7 @@ func (p *prometheus) ReportProcessedMetrics(mapping *MetricsMapping, r mb.Report
 }
 
 func getEvent(m map[string]common.MapStr, labels common.MapStr) common.MapStr {
-	hash := labels.String()
+	hash := LabelHash(labels.Flatten())
 	res, ok := m[hash]
 	if !ok {
 		res = labels

metricbeat/module/prometheus/collector/data.go

@@ -22,6 +22,7 @@ import (
 	"strconv"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/metricbeat/helper/prometheus"
 
 	dto "github.com/prometheus/client_model/go"
 )
@@ -34,7 +35,7 @@ type PromEvent struct {
 
 // LabelsHash returns a repeatable string that is unique for the set of labels in this event
 func (p *PromEvent) LabelsHash() string {
-	return p.labels.String()
+	return prometheus.LabelHash(p.labels)
 }
 
 func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent {