Tunable batch report behavior (istio#14301)

* added helm values to tune batch reporting behavior of telemetry data

* fix space formatting issue

* update dependencies from istio/api

* fix extra line

* move comment inside of {{- if }}

* move parameters back to top level values.yaml

* fix name of helm parameters

* fix comments

* fix comment

go.mod

@@ -182,7 +182,7 @@ require (
 	gopkg.in/stack.v1 v1.7.0 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 	gotest.tools v2.2.0+incompatible // indirect
-	istio.io/api v0.0.0-20190522004026-1dffc8d3d26d
+	istio.io/api v0.0.0-20190529184517-9f689ccd2715
 	istio.io/pkg v0.0.0-20190516214103-40b7bf4c7321
 	k8s.io/api v0.0.0-20190222213804-5cb15d344471
 	k8s.io/apiextensions-apiserver v0.0.0-20190221221350-bfb440be4b87

go.sum

@@ -601,11 +601,8 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 istio.io/api v0.0.0-20190515205759-982e5c3888c6/go.mod h1:hhLFQmpHia8zgaM37vb2ml9iS5NfNfqZGRt1pS9aVEo=
-istio.io/api v0.0.0-20190517041403-820986f2947c h1:b1WAnmPxFV6oU5LH6jiHBjDFa7rQgVJUdCCT6bmpawA=
-istio.io/api v0.0.0-20190517041403-820986f2947c/go.mod h1:hhLFQmpHia8zgaM37vb2ml9iS5NfNfqZGRt1pS9aVEo=
-istio.io/api v0.0.0-20190522004026-1dffc8d3d26d h1:ktu78abdVWQ9aIhocmEsm2pDeCwXL4ict6T0GGKLFuw=
-istio.io/api v0.0.0-20190522004026-1dffc8d3d26d/go.mod h1:hhLFQmpHia8zgaM37vb2ml9iS5NfNfqZGRt1pS9aVEo=
-istio.io/api v0.0.0-20190522135727-e29f1a9ce041 h1:Ho6DiXJ1lVOKsqeIv+eG6ZPHjvjFxo6WzKgDWf0BwOE=
+istio.io/api v0.0.0-20190529184517-9f689ccd2715 h1:SpIoxvzS72LrCPSIKBLIwVlxBkhXiZBbaSrgP/mgtS8=
+istio.io/api v0.0.0-20190529184517-9f689ccd2715/go.mod h1:hhLFQmpHia8zgaM37vb2ml9iS5NfNfqZGRt1pS9aVEo=
 istio.io/gogo-genproto v0.0.0-20190124151557-6d926a6e6feb/go.mod h1:eIDJ6jNk/IeJz6ODSksHl5Aiczy5JUq6vFhJWI5OtiI=
 istio.io/pkg v0.0.0-20190516214103-40b7bf4c7321 h1:zXJi3xLeMGtvewwN7LzzqGhpiAOw/pFrM+CPGd5Fowc=
 istio.io/pkg v0.0.0-20190516214103-40b7bf4c7321/go.mod h1:QuX5yn89LMkQdwuz0OyikOs3DBN45poAUNBAlOmFIvQ=

install/kubernetes/helm/istio/charts/mixer/values.yaml

@@ -45,6 +45,16 @@ telemetry:
       cpu: 4800m
       memory: 4G
 
+  # Set reportBatchMaxEntries to 0 to use the default batching behavior (i.e., every 100 requests). 
+  # A positive value indicates the number of requests that are batched before telemetry data 
+  # is sent to the mixer server
+  reportBatchMaxEntries: 100
+
+  # Set reportBatchMaxTime to 0 to use the default batching behavior (i.e., every 1 second). 
+  # A positive time value indicates the maximum wait time since the last request will telemetry data 
+  # be batched before being sent to the mixer server
+  reportBatchMaxTime: 1s
+
 podAnnotations: {}
 nodeSelector: {}
 tolerations: []

install/kubernetes/helm/istio/templates/configmap.yaml

@@ -19,6 +19,16 @@ data:
     disablePolicyChecks: true
     {{- end }}
 
+    {{- if .Values.mixer.telemetry.reportBatchMaxEntries }}
+    # reportBatchMaxEntries is the number of requests that are batched before telemetry data is sent to the mixer server
+    reportBatchMaxEntries: {{ .Values.mixer.telemetry.reportBatchMaxEntries }}
+    {{- end }}
+
+    {{- if .Values.mixer.telemetry.reportBatchMaxTime }}
+    # reportBatchMaxTime is the max waiting time before the telemetry data of a request is sent to the mixer server
+    reportBatchMaxTime: {{ .Values.mixer.telemetry.reportBatchMaxTime }}
+    {{- end }}
+
     # Set enableTracing to false to disable request tracing.
     enableTracing: {{ .Values.global.enableTracing }}
 

install/kubernetes/helm/istio/values.yaml

@@ -489,4 +489,4 @@ global:
   # Specifies whether helm test is enabled or not.
   # This field is set to false by default, so 'helm template ...'
   # will ignore the helm test yaml files when generating the template
-  enableHelmTest: false
+  enableHelmTest: false

pilot/pkg/networking/plugin/mixer/mixer.go

@@ -318,9 +318,11 @@ func buildTransport(mesh *meshconfig.MeshConfig, node *model.Proxy) *mccpb.Trans
 	}
 
 	res := &mccpb.TransportConfig{
-		CheckCluster:      buildUpstreamName(mesh.MixerCheckServer),
-		ReportCluster:     buildUpstreamName(mesh.MixerReportServer),
-		NetworkFailPolicy: networkFailPolicy,
+		CheckCluster:          buildUpstreamName(mesh.MixerCheckServer),
+		ReportCluster:         buildUpstreamName(mesh.MixerReportServer),
+		NetworkFailPolicy:     networkFailPolicy,
+		ReportBatchMaxEntries: mesh.ReportBatchMaxEntries,
+		ReportBatchMaxTime:    mesh.ReportBatchMaxTime,
 	}
 
 	return res

vendor/modules.txt

@@ -665,7 +665,7 @@ gopkg.in/square/go-jose.v2/json
 gopkg.in/stack.v1
 # gopkg.in/yaml.v2 v2.2.2
 gopkg.in/yaml.v2
-# istio.io/api v0.0.0-20190522004026-1dffc8d3d26d
+# istio.io/api v0.0.0-20190529184517-9f689ccd2715
 istio.io/api/mesh/v1alpha1
 istio.io/api/authentication/v1alpha1
 istio.io/api/mixer/v1/config/client