diff --git a/README.md b/README.md index ffb1b444102..fdf78845cb2 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ Error Set: [Developer Guide](/docs/DEVELOPER_GUIDE.md). ### Performance Tests +[KFServing benchmark test comparing Knative and Kubernetes Deployment with HPA](test/benchmark/README.md) [Performance Tests](https://docs.google.com/document/d/1ss7M3cx1qD1PVpTaKTu_Y3C80JJz4nvMZlIyuZutZoE/edit#) ### Contributor Guide diff --git a/pkg/controller/inferenceservice/controller_test.go b/pkg/controller/inferenceservice/controller_test.go index b0d68903041..a5a7731a626 100644 --- a/pkg/controller/inferenceservice/controller_test.go +++ b/pkg/controller/inferenceservice/controller_test.go @@ -25,8 +25,6 @@ import ( "knative.dev/pkg/network" - "github.com/kubeflow/kfserving/pkg/controller/inferenceservice/resources/knative" - "k8s.io/apimachinery/pkg/api/errors" "github.com/google/go-cmp/cmp" @@ -174,7 +172,6 @@ func TestInferenceServiceWithOnlyPredictor(t *testing.T) { "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", "autoscaling.knative.dev/maxScale": "3", "autoscaling.knative.dev/minScale": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": knative.DefaultQueueSideCarResourcePercentage, constants.StorageInitializerSourceUriInternalAnnotationKey: defaultInstance.Spec.Default.Predictor.Tensorflow.StorageURI, }, }, @@ -495,7 +492,6 @@ func TestInferenceServiceWithDefaultAndCanaryPredictor(t *testing.T) { "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", "autoscaling.knative.dev/maxScale": "3", "autoscaling.knative.dev/minScale": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": knative.DefaultQueueSideCarResourcePercentage, constants.StorageInitializerSourceUriInternalAnnotationKey: canary.Spec.Canary.Predictor.Tensorflow.StorageURI, }, }, @@ -1097,11 +1093,10 @@ func TestInferenceServiceWithTransformer(t *testing.T) { constants.KServiceComponentLabel: constants.Transformer.String(), }, Annotations: map[string]string{ - "autoscaling.knative.dev/target": "1", - "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", - "autoscaling.knative.dev/maxScale": "3", - "autoscaling.knative.dev/minScale": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": knative.DefaultQueueSideCarResourcePercentage, + "autoscaling.knative.dev/target": "1", + "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", + "autoscaling.knative.dev/maxScale": "3", + "autoscaling.knative.dev/minScale": "1", }, }, Spec: knservingv1.RevisionSpec{ @@ -1661,11 +1656,10 @@ func TestInferenceServiceWithExplainer(t *testing.T) { constants.KServiceComponentLabel: constants.Explainer.String(), }, Annotations: map[string]string{ - "autoscaling.knative.dev/target": "1", - "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", - "autoscaling.knative.dev/maxScale": "3", - "autoscaling.knative.dev/minScale": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": knative.DefaultQueueSideCarResourcePercentage, + "autoscaling.knative.dev/target": "1", + "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", + "autoscaling.knative.dev/maxScale": "3", + "autoscaling.knative.dev/minScale": "1", }, }, Spec: knservingv1.RevisionSpec{ diff --git a/pkg/controller/inferenceservice/reconcilers/knative/service_reconciler_test.go b/pkg/controller/inferenceservice/reconcilers/knative/service_reconciler_test.go index 2882b342eb2..fd9fe486f43 100644 --- a/pkg/controller/inferenceservice/reconcilers/knative/service_reconciler_test.go +++ b/pkg/controller/inferenceservice/reconcilers/knative/service_reconciler_test.go @@ -22,8 +22,6 @@ import ( "testing" "time" - "github.com/kubeflow/kfserving/pkg/controller/inferenceservice/resources/knative" - "github.com/google/go-cmp/cmp" "github.com/kubeflow/kfserving/pkg/apis/serving/v1alpha2" "github.com/kubeflow/kfserving/pkg/constants" @@ -130,7 +128,6 @@ func TestKnativeServiceReconcile(t *testing.T) { "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/target": "1", "internal.serving.kubeflow.org/storage-initializer-sourceuri": "gs://testuri", - "queue.sidecar.serving.knative.dev/resourcePercentage": knative.DefaultQueueSideCarResourcePercentage, }, }, Spec: knservingv1.RevisionSpec{ @@ -175,7 +172,6 @@ func TestKnativeServiceReconcile(t *testing.T) { "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/target": "1", "internal.serving.kubeflow.org/storage-initializer-sourceuri": "gs://testuri2", - "queue.sidecar.serving.knative.dev/resourcePercentage": knative.DefaultQueueSideCarResourcePercentage, }, }, Spec: knservingv1.RevisionSpec{ @@ -238,7 +234,6 @@ func TestKnativeServiceReconcile(t *testing.T) { "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/target": "1", "internal.serving.kubeflow.org/storage-initializer-sourceuri": "gs://testuri", - "queue.sidecar.serving.knative.dev/resourcePercentage": knative.DefaultQueueSideCarResourcePercentage, }, }, Spec: knservingv1.RevisionSpec{ diff --git a/pkg/controller/inferenceservice/resources/knative/service.go b/pkg/controller/inferenceservice/resources/knative/service.go index ea36b5d294e..74495837771 100644 --- a/pkg/controller/inferenceservice/resources/knative/service.go +++ b/pkg/controller/inferenceservice/resources/knative/service.go @@ -29,7 +29,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/serving/pkg/apis/autoscaling" - "knative.dev/serving/pkg/apis/serving" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" ) @@ -40,12 +39,6 @@ var serviceAnnotationDisallowedList = []string{ "kubectl.kubernetes.io/last-applied-configuration", } -const ( - // Set to 20% of the resource for main container, InferenceService defaults to 1CPU which is 200m for queue-proxy - // https://github.com/knative/serving/blob/1d263950f9f2fea85a4dd394948a029c328af9d9/pkg/reconciler/revision/resources/resourceboundary.go#L30 - DefaultQueueSideCarResourcePercentage = "20" -) - type ServiceBuilder struct { inferenceServiceConfig *v1alpha2.InferenceServicesConfig credentialBuilder *credentials.CredentialBuilder @@ -393,9 +386,6 @@ func (c *ServiceBuilder) buildAnnotations(metadata metav1.ObjectMeta, minReplica annotations[autoscaling.MaxScaleAnnotationKey] = fmt.Sprint(maxReplicas) } - if _, ok := annotations[serving.QueueSideCarResourcePercentageAnnotation]; !ok { - annotations[serving.QueueSideCarResourcePercentageAnnotation] = DefaultQueueSideCarResourcePercentage - } // User can pass down scaling target annotation to overwrite the target default 1 if _, ok := annotations[autoscaling.TargetAnnotationKey]; !ok { if parallelism == 0 { diff --git a/pkg/controller/inferenceservice/resources/knative/service_test.go b/pkg/controller/inferenceservice/resources/knative/service_test.go index 0fcec86004d..9be797786ed 100644 --- a/pkg/controller/inferenceservice/resources/knative/service_test.go +++ b/pkg/controller/inferenceservice/resources/knative/service_test.go @@ -46,8 +46,7 @@ var isvc = v1alpha2.InferenceService{ Name: "mnist", Namespace: "default", Annotations: map[string]string{ - constants.InferenceServiceGKEAcceleratorAnnotationKey: "nvidia-tesla-t4", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, + constants.InferenceServiceGKEAcceleratorAnnotationKey: "nvidia-tesla-t4", }, }, Spec: v1alpha2.InferenceServiceSpec{ @@ -100,7 +99,6 @@ var defaultService = &knservingv1.Service{ "autoscaling.knative.dev/target": "1", "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/maxScale": "3", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, constants.InferenceServiceGKEAcceleratorAnnotationKey: "nvidia-tesla-t4", constants.StorageInitializerSourceUriInternalAnnotationKey: isvc.Spec.Default.Predictor.Tensorflow.StorageURI, }, @@ -149,7 +147,6 @@ var canaryService = &knservingv1.Service{ "autoscaling.knative.dev/target": "1", "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/maxScale": "3", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, constants.InferenceServiceGKEAcceleratorAnnotationKey: "nvidia-tesla-t4", constants.StorageInitializerSourceUriInternalAnnotationKey: "s3://test/mnist-2/export", }, @@ -195,8 +192,7 @@ func TestInferenceServiceToKnativeService(t *testing.T) { Name: "mnist", Namespace: "default", Annotations: map[string]string{ - constants.InferenceServiceGKEAcceleratorAnnotationKey: "nvidia-tesla-t4", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, + constants.InferenceServiceGKEAcceleratorAnnotationKey: "nvidia-tesla-t4", }, }, Spec: v1alpha2.InferenceServiceSpec{ @@ -273,7 +269,6 @@ func TestInferenceServiceToKnativeService(t *testing.T) { "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/target": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, }, }, Spec: knservingv1.RevisionSpec{ @@ -334,7 +329,6 @@ func TestInferenceServiceToKnativeService(t *testing.T) { "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/target": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, }, }, Spec: knservingv1.RevisionSpec{ @@ -396,7 +390,6 @@ func TestInferenceServiceToKnativeService(t *testing.T) { "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", "autoscaling.knative.dev/minScale": "1", "autoscaling.knative.dev/target": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, }, }, Spec: knservingv1.RevisionSpec{ @@ -434,7 +427,6 @@ func TestInferenceServiceToKnativeService(t *testing.T) { "autoscaling.knative.dev/target": "2", constants.StorageInitializerSourceUriInternalAnnotationKey: "test", "kubectl.kubernetes.io/last-applied-configuration": "test2", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, }, }, Spec: v1alpha2.InferenceServiceSpec{ @@ -472,7 +464,6 @@ func TestInferenceServiceToKnativeService(t *testing.T) { "autoscaling.knative.dev/minScale": "1", "sourceName": "srcName", "prop1": "val1", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, }, }, Spec: knservingv1.RevisionSpec{ @@ -613,11 +604,10 @@ func TestTransformerToKnativeService(t *testing.T) { constants.KServiceComponentLabel: constants.Transformer.String(), }, Annotations: map[string]string{ - "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", - "autoscaling.knative.dev/target": "1", - "autoscaling.knative.dev/minScale": "1", - "autoscaling.knative.dev/maxScale": "3", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, + "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", + "autoscaling.knative.dev/target": "1", + "autoscaling.knative.dev/minScale": "1", + "autoscaling.knative.dev/maxScale": "3", }, }, Spec: knservingv1.RevisionSpec{ @@ -660,11 +650,10 @@ func TestTransformerToKnativeService(t *testing.T) { constants.KServiceComponentLabel: constants.Transformer.String(), }, Annotations: map[string]string{ - "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", - "autoscaling.knative.dev/target": "1", - "autoscaling.knative.dev/minScale": "2", - "autoscaling.knative.dev/maxScale": "4", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, + "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", + "autoscaling.knative.dev/target": "1", + "autoscaling.knative.dev/minScale": "2", + "autoscaling.knative.dev/maxScale": "4", }, }, Spec: knservingv1.RevisionSpec{ @@ -807,10 +796,9 @@ func TestExplainerToKnativeService(t *testing.T) { constants.KServiceComponentLabel: constants.Explainer.String(), }, Annotations: map[string]string{ - "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", - "autoscaling.knative.dev/minScale": "1", - "autoscaling.knative.dev/target": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, + "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", + "autoscaling.knative.dev/minScale": "1", + "autoscaling.knative.dev/target": "1", }, }, Spec: knservingv1.RevisionSpec{ @@ -853,10 +841,9 @@ func TestExplainerToKnativeService(t *testing.T) { constants.KServiceComponentLabel: constants.Explainer.String(), }, Annotations: map[string]string{ - "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", - "autoscaling.knative.dev/minScale": "1", - "autoscaling.knative.dev/target": "1", - "queue.sidecar.serving.knative.dev/resourcePercentage": DefaultQueueSideCarResourcePercentage, + "autoscaling.knative.dev/class": "kpa.autoscaling.knative.dev", + "autoscaling.knative.dev/minScale": "1", + "autoscaling.knative.dev/target": "1", }, }, Spec: knservingv1.RevisionSpec{ diff --git a/test/benchmark/README.md b/test/benchmark/README.md new file mode 100644 index 00000000000..8d3bcdf0a78 --- /dev/null +++ b/test/benchmark/README.md @@ -0,0 +1,289 @@ +# Benchmark + +This benchmark focus on testing KFServing performance with and without Knative queue proxy/activator on the request path. + +* Knative queue proxy does the following for the KFServing main container. + - Enforces concurrency level for the pod + - Emit metrics for autoscaling(KPA) + - Timeout enforcement + - Readiness probe + - Queue limiting + - Distributed tracing + - Graceful shutdown handling + +* Knative activator buffers the requests while pods are scaled down to zero and report metrics to autoscaler. The activator +also effectively acts as a load balancer which distributes the load across all the pods as they become available in a way that +does not overload them with regards to their concurrency settings. So it protects the app from burst so you do not see messages +queuing in the user pods. + +## Environment Setup +- K8S: v1.14.10-gke.36(8 nodes n1-standard) +- Istio: 1.1.6 +- Knative: 0.11.2 +- KFServing: master(with fix for https://github.com/kubeflow/kfserving/issues/844) + +Note that `v1.14.10-gke.36` suffers the [CFS throttling bug](https://github.com/kubernetes/kubernetes/issues/67577), +and `1.15.11-gke.15` includes the CFS throttling fix. + +## Benchmarking + +### Results on KFServing SKLearn Iris Example +- Create `InferenceService` +```bash +kubectl apply -f ./sklearn.yaml +``` +- Create the input vegeta configmap +```bash +kubectl apply -f ./sklearn_vegeta_cfg.yaml +``` +- Create the benchmark job using [vegeta](https://github.com/tsenart/vegeta) +Note that you can configure pod anti-affinity to run vegeta on a different node on which the inference pod is running. +```bash +kubectl create -f ./sk_benchmark.yaml +``` + +#### CC=8 With queue proxy and activator on the request path +Create an `InferenceService` with `ContainerCurrency`(cc) set to 8 which is equal to the number of cores on the node. +```yaml +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "sklearn-iris" +spec: + default: + parallelism: 8 # CC=8 + predictor: + sklearn: + storageUri: "gs://kfserving-samples/models/sklearn/iris" +``` + +| QPS/Replicas | mean | p50 | p95 | p99 | Success Rate | +| --- | --- | --- | --- | --- | --- | +| 5/s minReplicas=1 | 6.213ms | 5.915ms | 6.992ms | 7.615ms | 100% | +| 50/s minReplicas=1 | 5.738ms | 5.608ms | 6.483ms | 6.801ms | 100% | +| 500/s minReplicas=1 | 4.083ms | 3.743ms | 4.929ms | 5.642ms | 100% | +| 1000/s minReplicas=1 | 398.562ms | 5.95ms | 2.945s | 3.691s | 100% | + +#### Raw Kubernetes Service(Without queue proxy and activator on the request path) +- Update the SKLearn Iris `InferenceService` with following yaml to use HPA +```yaml +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "sklearn-iris" + annotations: + autoscaling.knative.dev/class: hpa.autoscaling.knative.dev + autoscaling.knative.dev/metric: cpu + autoscaling.knative.dev/target: "80" +spec: + default: + predictor: + sklearn: + storageUri: "gs://kfserving-samples/models/sklearn/iris" +``` +```bash +kubectl apply -f ./sklearn_hpa.yaml +``` +- Setup virtual service to go directly to the private service to bypass the Knative Activator and queue-proxy, change the benchmark +test target url host to `sklearn-iris-raw.default.svc.cluster.local`. +```yaml +apiVersion: v1 +kind: Service +metadata: + name: sklearn-iris-raw +spec: + externalName: cluster-local-gateway.istio-system.svc.cluster.local + sessionAffinity: None + type: ExternalName +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: sklearn-iris-raw +spec: + gateways: + - knative-serving/cluster-local-gateway + hosts: + - sklearn-iris-raw.default.svc.cluster.local + http: + - match: + - authority: + regex: ^sklearn-iris-raw\.default(\.svc(\.cluster\.local)?)?(?::\d{1,5})?$ + gateways: + - knative-serving/cluster-local-gateway + uri: + regex: ^/v1/models/[\w-]+(:predict)? + route: + - destination: + host: sklearn-iris-predictor-default-xt264-private.default.svc.cluster.local #this is the private service to user container + port: + number: 80 + weight: 100 +``` + +| QPS/Replicas | mean | p50 | p95 | p99 | Success Rate | +| --- | --- | --- | --- | --- | --- | +| 5/s Replicas=1 | 2.673ms | 2.381ms | 4.352ms | 5.966ms | 100% | +| 50/s Replicas=1 | 2.188ms | 2.117ms | 2.684ms | 3.02ms | 100% | +| 500/s Replicas=1 | 1.376ms | 1.283ms | 1.713ms | 2.205ms | 100% | +| 1000/s Replicas=1 | 7.969s | 8.658s | 16.669s | 20.307s | 93.72% | + +So you can see that queue-proxy and activator adds 2-3 millisecond overhead, but you get the advantage of KPA and +smart load balancing. For this example we do not see much benefits because the request takes only 1-2 ms to process, +however you can see the obvious advantage when request volume goes to 1000/s and KPA reacts faster and performs better +than HPA. + +### Results on KFServing with TFServing Flower Example +- Create `InferenceService` +```bash +kubectl apply -f ../docs/samples/tensorflow/tensorflow.yaml +``` +- Create the input vegeta configmap +```bash +kubectl apply -f ./tf_vegeta_cfg.yaml +``` +- Create the benchmark job using [vegeta](https://github.com/tsenart/vegeta) +Note that you can configure pod anti-affinity to run vegeta on a different node on which the inference pod is running. +```bash +kubectl create -f ./tf_benchmark.yaml +``` + +#### CC=0 +- Create `InferenceService` with default `ContainerConcurrency` set to 0 which is unlimited concurrency, activator in this case just pass +through and you would still expect requests queued on user container in case of request overload. +```yaml +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "flowers-sample" +spec: + default: + predictor: + tensorflow: + storageUri: "gs://kfserving-samples/models/tensorflow/flowers + resources: + requests: + cpu: "4" + memory: 2Gi + limits: + cpu: "4" + memory: 2Gi +``` + +```bash +kubectl apply -f ./tf_flowers.yaml +``` + +| QPS/Replicas | mean | p50 | p95 | p99 | Success Rate | +| --- | --- | --- | --- | --- | --- | +| 1/s minReplicas=1 | 110.54ms | 110.343ms | 116.116ms | 117.298ms | 100% | +| 5/s minReplicas=1 | 133.272ms | 131.242ms | 148.195ms | 153.291ms | 100% | +| 10/s minReplicas=1 | 946.376ms | 127.961ms | 4.635s | 6.934s | 100% | + +#### CC=1 +- Create `InferenceService` with `ContainerConcurrency` set to 1, activator respects container queue limit 1 so that requests do +not get queued on user pods and activator chooses to route the requests to the pods which have capacity. + +```yaml +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "flowers-sample" +spec: + default: + predictor: + parallelism: 1 #CC=1 + tensorflow: + storageUri: "gs://kfserving-samples/models/tensorflow/flowers + resources: + requests: + cpu: "4" + memory: 2Gi + limits: + cpu: "4" + memory: 2Gi +``` + +| QPS/Replicas | mean | p50 | p95 | p99 | Success Rate | +| --- | --- | --- | --- | --- | --- | +| 1/s minReplicas=1 | 103.766ms | 102.869ms | 111.559ms | 116.577ms | 100% | +| 5/s minReplicas=1 | 117.456ms | 117.117ms | 122.346ms | 126.139ms | 100% | +| 10/s minReplicas=1 | 702.249ms | 111.289ms | 3.469s | 3.831s | 100% | + + +So here you can see that with CC=1, when you send one request at a time the latency does not make much different with CC=0 or CC=1. +However when you send more concurrent requests you start to notice pronounced result when CC=1 because activator starts to take effect and you +will observe better tail latency at p95 and p99 thanks to Knative activator [smarter load balancing](https://github.com/knative/serving/issues/5692) than random load balancing. + +#### Raw Kubernetes Service(Without queue proxy and activator) +```yaml +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "flowers-sample-hpa" + annotations: + autoscaling.knative.dev/class: hpa.autoscaling.knative.dev + autoscaling.knative.dev/metric: cpu + autoscaling.knative.dev/target: "60" +spec: + default: + predictor: + tensorflow: + storageUri: "gs://kfserving-samples/models/tensorflow/flowers + resources: + requests: + cpu: "4" + memory: 2Gi + limits: + cpu: "4" + memory: 2Gi +``` +Setup virtual service to bypass the knative proxy and update vegeta config target URL to +`http://flowers-sample-raw.default.svc.cluster.local/v1/models/flowers-sample-hpa:predict` + + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: flowers-sample-raw + namespace: default +spec: + externalName: cluster-local-gateway.istio-system.svc.cluster.local + sessionAffinity: None + type: ExternalName +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: flowers-sample-raw +spec: + gateways: + - knative-serving/cluster-local-gateway + hosts: + - flowers-sample-raw.default.svc.cluster.local + http: + - match: + - authority: + regex: ^flowers-sample-raw\.default(\.svc(\.cluster\.local)?)?(?::\d{1,5})?$ + gateways: + - knative-serving/cluster-local-gateway + uri: + regex: ^/v1/models/[\w-]+(:predict)? + route: + - destination: + host: flowers-sample-hpa-predictor-default-95bbz-private.default.svc.cluster.local #this is the private service to user container + port: + number: 80 + weight: 100 +``` + +| QPS/Replicas | mean | p50 | p95 | p99 | Success Rate | +| --- | --- | --- | --- | --- | --- | +| 1/s Replicas=1 | 129.143ms | 112.853ms | 118.143ms | 128.557ms | 100% | +| 5/s Replicas=1 | 127.947ms | 127.549ms | 132.171ms | 135.801ms | 100% | +| 10/s Replicas=1 | 5.461s | 5.087s | 12.992s | 14.587s | 100% | + +This experiment runs the `InferenceService` using HPA with average target utilization 80% of CPU and calls directly to Kubernetes Service bypassing +the Knative queue proxy and activator. You can see that KPA reacts faster with the load and performs better than HPA for both low latency and high latency +requests. diff --git a/test/benchmark/sk_benchmark.yaml b/test/benchmark/sk_benchmark.yaml new file mode 100644 index 00000000000..c2aee232260 --- /dev/null +++ b/test/benchmark/sk_benchmark.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: sklearn-load-test +spec: + backoffLimit: 6 + parallelism: 1 + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - args: + - vegeta -cpus=1 attack -duration=10m -rate=5/1s -targets=/var/vegeta/cfg + | vegeta report -type=text + command: + - sh + - -c + image: peterevans/vegeta:latest + imagePullPolicy: Always + name: vegeta + resources: + requests: + cpu: "1" + memory: 1Gi + volumeMounts: + - mountPath: /var/vegeta + name: vegeta-cfg + restartPolicy: Never + volumes: + - configMap: + defaultMode: 420 + name: vegeta-cfg + name: vegeta-cfg +--- + diff --git a/test/benchmark/sklearn.yaml b/test/benchmark/sklearn.yaml new file mode 100644 index 00000000000..f99a9721413 --- /dev/null +++ b/test/benchmark/sklearn.yaml @@ -0,0 +1,10 @@ +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "sklearn-iris" +spec: + default: + parallelism: 8 # CC=8 + predictor: + sklearn: + storageUri: "gs://kfserving-samples/models/sklearn/iris" diff --git a/test/benchmark/sklearn_hpa.yaml b/test/benchmark/sklearn_hpa.yaml new file mode 100644 index 00000000000..0c87959ca70 --- /dev/null +++ b/test/benchmark/sklearn_hpa.yaml @@ -0,0 +1,13 @@ +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "sklearn-iris" + annotations: + autoscaling.knative.dev/class: hpa.autoscaling.knative.dev + autoscaling.knative.dev/metric: cpu + autoscaling.knative.dev/target: "80" +spec: + default: + predictor: + sklearn: + storageUri: "gs://kfserving-samples/models/sklearn/iris" diff --git a/test/benchmark/sklearn_vegeta_cfg.yaml b/test/benchmark/sklearn_vegeta_cfg.yaml new file mode 100644 index 00000000000..9671c1efea3 --- /dev/null +++ b/test/benchmark/sklearn_vegeta_cfg.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +data: + cfg: | + POST http://sklearn-iris.default.svc.cluster.local/v1/models/sklearn-iris:predict + @/var/vegeta/payload + payload: | + { + "instances": [ + [6.8, 2.8, 4.8, 1.4], + [6.0, 3.4, 4.5, 1.6] + ] + } +kind: ConfigMap +metadata: + annotations: + name: vegeta-cfg diff --git a/test/benchmark/tf_benchmark.yaml b/test/benchmark/tf_benchmark.yaml new file mode 100644 index 00000000000..15ae07f2b73 --- /dev/null +++ b/test/benchmark/tf_benchmark.yaml @@ -0,0 +1,35 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: tf-load-test +spec: + backoffLimit: 6 + parallelism: 1 + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - args: + - vegeta -cpus=5 attack -duration=1m -rate=5/1s -targets=/var/vegeta/cfg + | vegeta report -type=text + command: + - sh + - -c + image: peterevans/vegeta:latest + imagePullPolicy: Always + name: vegeta + resources: + requests: + cpu: "1" + memory: 1Gi + volumeMounts: + - mountPath: /var/vegeta + name: tf-vegeta-cfg + restartPolicy: Never + volumes: + - configMap: + defaultMode: 420 + name: tf-vegeta-cfg + name: tf-vegeta-cfg diff --git a/test/benchmark/tf_flowers.yaml b/test/benchmark/tf_flowers.yaml new file mode 100644 index 00000000000..49fd81924f4 --- /dev/null +++ b/test/benchmark/tf_flowers.yaml @@ -0,0 +1,17 @@ +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "flowers-sample" +spec: + default: + predictor: + parallelism: 1 #CC=1 + tensorflow: + storageUri: "gs://kfserving-samples/models/tensorflow/flowers" + resources: + requests: + cpu: "4" + memory: "2Gi" + limits: + cpu: "4" + memory: "2Gi" diff --git a/test/benchmark/tf_flowers_hpa.yaml b/test/benchmark/tf_flowers_hpa.yaml new file mode 100644 index 00000000000..df4e464faec --- /dev/null +++ b/test/benchmark/tf_flowers_hpa.yaml @@ -0,0 +1,20 @@ +apiVersion: "serving.kubeflow.org/v1alpha2" +kind: "InferenceService" +metadata: + name: "flowers-sample-hpa" + annotations: + autoscaling.knative.dev/class: hpa.autoscaling.knative.dev + autoscaling.knative.dev/metric: cpu + autoscaling.knative.dev/target: "60" +spec: + default: + predictor: + tensorflow: + storageUri: "gs://kfserving-samples/models/tensorflow/flowers" + resources: + requests: + cpu: "4" + memory: "2Gi" + limits: + cpu: "4" + memory: "2Gi" diff --git a/test/benchmark/tf_vegeta_cfg.yaml b/test/benchmark/tf_vegeta_cfg.yaml new file mode 100644 index 00000000000..b354e912356 --- /dev/null +++ b/test/benchmark/tf_vegeta_cfg.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +data: + cfg: | + POST http://flowers-sample.default.svc.cluster.local/v1/models/flowers-sample:predict + @/var/vegeta/payload + payload: | + { + "instances":[ + { + "image_bytes":{ + "b64":"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAErASsDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwC9A42ir9vA0nOOKxYJhkDqe1bNvO0ZAYdaIsC8LLjOwH60yWDAwY1/75qzDcDAz0qfhl55BqxGE1pCzZwVPt0qJ7MgZQbh7da1Z7bncBVQgoaVhlGFvKlBIwOhqxPFxkdKmdY5xiQYP94daaqtGPKkO5P4X/pU2AoKMMQatWv+tAPXpTJ4ipyBTVYqwYHBFTezA1ivHNRsuRU1tOlymOBIOo9aVoyGNaCIEHanEEEMKXbg07BAx2NICXO5KrvwcVPEcDFRyD5qTYDYhuPuKnA4waitxmQj1FWGX9Ka2ArODzUXU5qxIM81AODzUtjGzHMfvVRcl6mmOMio4V3PSAtwjBUd60l+6DVCMAzH2q6D8v0qo7CIJ3xmsqQ8kmtC5YAVmyctntSbGRkDOT0qWMFyABUWNzD0q5EuxM9zQgJQAqgCkJxS9vemMasA3c8CpFPHNRBgBkinBvSpuBMGxRnPWo1561IOlMBQMEU2R8DFKW2rk1XdsmgCN+TmqskuHIqeUhVNZMkoZyckZqQILTi5UntzWtHMOVbpWQh2zCr6jIBpRGzUjl2jBPHY1chuSODyKx4pOzdKnVyh68VYjbDBlyvSq88G4bhVeG4Kkc8HrV3eGUEVQjLkUr+FRmQgYzV+aMODxzWdIpU0mMerh1wahdCpPvTN21gQamB3jB+qn1rOQDI5GjcMvBFbdvMt1FkfeHWsJhzU1pcG3nDZ4PWlGVgNd4+MigL8uKscMgdeVNRsAORVsRGFwc1G45qfKg/MM/U0jLG3RQPxNS2BCh2OG9DVxwM57GqxRQc8j9asp80I5zjiiIyu64zVdhxVtwMVVak2BUlOTUlumATTXXmpPux0r6AS2vLv7GrLNtFVbM/K596knbgGqT0AqXLZeqbgsRU8x96hJzgCk2A+JPmA61PA4mUSL9wk7fcetULtmEMdvGSJrltgI6hQMsfwH6kVqRIsUaqgAVQAAOwFUgEJ7UwDOc1Ky55/OmtgcCi4EZ6UqqSc0Hk4p46igB44pQaaM5NI7hVx3qkA2V8nHaoAdzE9hTZHOMd6ZczfZoQq/fNDArahcgAxLyf4iKzs0OxJ5696ZUDQP97NaVsdyg+1IPszHlFzU8SRg4jGB6VSQh3linp02mpQm5enNJs9aoBoynfirMFwVOD0qADjDUn3W9qANIsGGQeKqXCK3PekjlIOCeKfJyN1AGXIMZFNik6xscc5U+hqxMgbPrVFwVas2BezvXOMOPvCo2GD7UyOXOG/iHX3p8hGzdn6Vm0M0rG8ZLYxtzz8pp0lyx/iNZUMpzzVkturURKZGP8AEfzpRMw6Nmq5HvTMspz1pAaUVzzhjiptxjPmRnjuKyBNzzxU8NwUbDcqaXoBreYJU3L+VVn5zTEcRvkHKNUjcE4qZdwITyabK3yGpG4GaqzN+7qG9ALNicwn/eNE75UgU2zb/RQfc0krY4rS+gFZgcc0iKM+9Kc81FcI727QxnDyjbu9AepoWrAZpv8Apt7NqB5jA8mAeig/MfxOPyFa4HFQ20KW8KQxrhEUKB7VYXFWAvlkhSDx1PvUchwSAKlD7Uyep6CoS3UnrU9QGHg0DJ5xSb8mjdjvVAOZ9oqs75JOaJX3Hg1GBmmAKRuLt0Xms24lMshbt2qxezgDyEPuxqkxpNjImo4pGOOarmbk0gJvMINWIbp42BB6VBLC0Z9qjVsGjYDqrWVJ4w6n6j0qcxbh71ztndtbyBlPHcetdLayx3Me+Ns+o7irTuJkDRDvwaYVIODWg0IYc9agkgBGDTEUyCv0pwc4wac8ZTg9KjYFRSAil6ZxVOYZFXGPBBqpKKljIFJB61KzFlqJhTkbPBrO+oxysR0qwrkjk1Wxg1IoPBJ4PpSc7BYkOfU0zzHXvke9WNuFBHQ1A/BrKVRoaQm/dweDShyOOtRZB+tAPHNSq6HYv203/LNuh6VeVvkweq1ioxB+lacUm5Nw7jBrVTUoktEsp+SqNw2F4q1I3yCqM5yQKwchpF62Yi0/GkcHgmkh4gAoOSK1UxEfVuKdbKZH8zseF+nrTGO0qo6scVciXgYrWG1xDxwcUm/GQKc3FVS3J5qmwHmUkknoKYXJGaZuBzTd1JMB4PBzxUUkhPApWcnpTFUk1SAdGrOcVW1S/TTbbIwZW4Rff1rQXbEmSefSqC6bHPdNd3Q86U/dDfdQegFUBkWYurnlI2YseWPetSPSZW5llC+yjJrUVABjt6CnHihJICkulWqcsGf/AHjUwt4QMCGMD/cFSM4HWm+YKq4HO/aAww3NRuqtypxTNpFIMisrjFDFDg1ctbySCQSRsQf5/Wqm4MOaT6GlewHY2OrQXWFciOT0J4NaDRq68/nXBLIRwa0bTVLi2wBIWX0bmqU+4rHSSQcFW5HY1TkiKZBGRUtrq8NyNr8N6VaYJIvysCPar0YjGdMDOeKrSL6ng1pXFuUJxyDVCSMgH0qGMqMvao84NTOp61XYkHmspDRYXBxnpSFjG2DyKbEdwK96eR5ilT95elZSKRYglBHlseD0NJKpU4NUlk2nB6VcjlEq7HPzDofWsJTvox2KzcZojbccGnSrgkVCpIkFcdaorblJFg8VctHzlapBs1Ztf9bTw+K01CUS07dvaqMh3TqKulSSTjPHWqCgGdnzkKOtYLGxbDkZoxnEYqR4j5e4HPqKrwncAO2M1X1jVRp+lXFxn7inHu3QCu6E3JKzIaC1lF3qk+05jtgIh/vnlj+AwPzrXUYArnvB0LLoUMshJknZpXJ7kmuhLAZxXcpIgimYjPNVd3HWpJnLHFQgcYp81xjht7U4AGkGFpDPt4QZNaIQ9YiRnoPemng4Tk+tIFdzmRjj0p5dEHFWmAqRfxOeakyAOwquZ93Sk5PJouBM0uKjMrHpUeRn5qQydhRcB3Pc0vHrUYJanbT6UwMX6ikKelafA/gX8qRljPWJfwzScAMsxjNJtx9K0Ht42HykqffkVA8DpzjI9RzUOLQ7lXbijDdRU2zIppX3qRipJ0z1rQttQkj4Ylh655FZjLzQMryDRewHUwXyTphiDnof8aZcQYO5eRXPw3BQ5U4Na1tf5Xa3I9KtSuTYgkjK5x0qpIpwM9K2HRJlLIefSqEsRGR+lRJAUVcxuCO1WpDlROn4iqknTp0p1tPtYo3KNXJOdtGWlcllUMBIvRuv1oGVOD1FOWIiRIz91zgH0qS6VoNwaMgKeJQMgfWvExGK960TeMRpPmqMnD4796JIQsf7s5lGAQT61YeSOS8SFwFkH3WHRuKq6fFJdajMZ3aMRtgqByT269q4nUnJXb2KskMjUiIS7sl32hfTtWhaArIQ42kdaaY/7PvzG6+dFMSWfHC91+nvS2e+Sab7QORNsUZwG7jn0xmpc3a6FYnDpC4lDEl/4G6DtVa5UqRDEADKTj2Hqas3Nkz6hGZGxCFJLL/FzwKluIykkQgQtC53N3Kn0+maxUrNMvRoiEZWPapAJ71keItPkvdPYiNnjg+cQjOZT/8AW61uOY1cGUcryFzUgkZ0JVsDtXfDGWmm9kZuGhX0qJLXT4IACuyMfKe1WZJQFOKiKlSCWycfnUUjjua9ali41PhZk42EbLHNISFFM83nA5pVXPLV3Ql2IaDBf2FOykQycUySUIMDrVZi0h5NbJiJnuGc4WkVWP3qYo29OtSKrt34q0wHhgo4GaCx/CnLCw561OkJP8NWIrBd/apVtj3qysI74FSZjTimBElsO1S+QKQzf3cCk3v60AUiqEcVG0R7DNPIZT92kLY5yRWgFd0I7UwMyGrLHPYGoXQHkVLAYYUlGUwj/oarMhRyrqQam3FDU6yRzoEl7dG7ioaTAzyuR0phQ+tXJreSLn7yHow6GoNhas2h3K5UA5zSrIUI5qQx5qIoBwRUMZehvMEZbB7GrS3Mdy/lMQk3YH+L6HvWJnHHpSs0c0ZinBKeoOCp9Qe1Tz2CxcvY2jY8VXgYMrhhhf73pTY728geO3ukW9tXztuPusvfDe9aECQmMqE3I/OD1FeVmFaKVlua049yNLooVjLD5MMcjORmrZN3LfmHzVjg2bn3LkMvp9TVeS6htbNXSNTk5RiPSrdhdx3ds0lztIcHJHHHt714cr/FY2JTbwGzDJ80kBym48/5xTZHzLFLLEygryw43+lFvJa/YZF52AZEh5IIpkN6k2yCcOcHdtIxzUaktl10gup02zMqlcFRyM0omRElhk25QYGBzu9qqLav9rl+ySII+q7mzz6UxHiXzYbkL9oHzGQHnd2x7VLj0Fc0Fml+y5kjbY2OT/CakaRSEMY/eE7do71krqBkgWNllCzMOdp/P0q35aQPEYHd9wKlSMnjvUuFtykyeW1ju51WQsjIm38e1V5LWRSiGcDy+Hx/ETzmpxK0kgl2ERYwzHru+lJe2pn8trc4kzyC3BHrSTa0ZRG7IkQIbcvTJ65qsUeZ2H3VXqT6065kkSfyIrZiQMFmHHHU06TKWyq7ZbGSfU16GB91uUnotTOYxIVU/eyaSRivFAxgHOaGw3yn8K9+hVhWjeBi01uQFWY05YSe9L5wXjbUiTqeq4+ldcSWSRwkdRmp1RVGW4qITf3TxSg7jljWqJJRJGvTrSGZj0OBTfK3Hini3brV2Ab5jNxTgCetSCE9+DR5ZxinYBAqjmjzVpPKOOtHlU7AVRLwOhoyHFZyuw6VMsz9aq4FghajYYHBpPNzzRvDD0pMCB8HtioiQOhqV1Peq0kZBz2qWBbgu2i44ZT1U9DVwWttdjdAxjfvH/hWGWZDx0qWO4dCGBII7ip5h2L82nlTgvtP+0KrPZSjkbH+jVft9VhnTyrtQQeN4qO7025C+bp06TL/AM8pDj8mH9RSaT2AyJo2X7yEH3FVJJFUdeadN4hlsJvK1Kxnt+cbiNy/nTzf6XerkBTnupK1yVdFcpDdME8935SofKP3s9vetcmCxTZcDcm4/Oh5APrVWK4kuA0dtHjbznODTzAbiaGV0EinO5Dkfn+NfN4ibqTvLRHTHRE8losVspkKT23VY+4z3z9Ka8FvayRyQiT7G/OGHCmp44omsnW5LbsHdGrcLjgfXtQdQVdOkhYeblNqoozx9K57vYTIbmWAT2zoGUOSGUDCk9jS3Fwl7fpsuFjZUAdgM5PSobm/hmsIbZnVmaQEJjgge/arN1FHdxWoh8qBgx3N/s/TuelVta5DZWN79gmliMgaVMkcEBqnhube4s5FmTMjjcGbruPpisvVIn0u5jE9wkz3PAcLjaKkb7Pp8kEkL7lkByHP3SMcj3q+RWTW7Iua0OpLJYzQyZ+VMFAMMD2xn3plnfva3Dw3aeXMg4UntjrVC9vXk8m+ijXbG20kA7m96fNqCSz29y6EmL5Hcp0zjGah0tNtylI14dTVpZHPIz90jk1Na5aLzvNBc/MExwB6fWq7zW63cExVBI8fGO/PFSXksUcgMZbMv+s2jhPeudxWyNUyO7vfOuo0BYKAGc+vtUVwr3upCFNywxrukfH8q02mha0kAjUqqZB6Z46VAl15kGyFQCVOBn+tVSqcmtgavoQtDDgorlSOnOahkzCu6QjaOd3bFPQmGVEeAmM43yZ6+uKknaPa8IAdG4OfSvewlanJ6JL0MZJmb9ttWOVcyeyKT/TFC3Fy5/0fTZCv964kWIfhjcf0qyMRjCKFHsMU1mYmvWi0ZsehuMfvPIiOekbGT9SF/lUhl2+hqDDkU3Yx5NbJiLH2kjocU8XLf3jVUITRsNXcRdE7H+Ol85/7xqmEYdDThuHencC557g/epftL+oqllqTLU7gQKvNPCkHBNOVkIG5amVI2Aw2KYEITqKTYQfSrnkHqMGlMB/u8UgKTKfc00oSORV4REdRQbfnikwMxoh0IqJodp+WtY24I55NNNvj+GpaAx2hYcg0+C5ntmzHIw9uoNXmtjk4FQSQbe1ZO62KRZTVbe7Qw30KAHuRlTUN5YlSj2RhEWeU8sFT9DVCZAB2qvDcSQMfLkIH90nINcmJqtQa6lRjqaUTNNI1rtWJ8E7umfUVIoffHapcA7Ry5GCB/WoLC4juhKZGjSUHaoc4/HPTFVPImsbo3VyywhMjy2OSwPpivm3FuTT3OnoXLuAxTC3FzuWYFtxGDwelN06eHTmfjYe4cckVMyiOSO4nlWSXd8qfwjIqS6uYZb+1V9nmxhjz9OKm+ljNmJetBJqdw8mYlAHl4UqPekt5tQnuVeOESW6ggEN6VY8Tajb6nHBYxnE5bJbHQVDpMdxbXCWECmYjJGDwfqa6Uv3d2tfP8yGJBqcWpTvHdw7Y0G1S33vfH5VB/ZN5GrtegPDJ8lsQ3IPbP1qxc6PEbGW4Fw0FxHlnjK8euKqy67P/AGfBEkb/ALyQYJGFBHaqjr/D2/IhmlYvcaVcpHeRqpC5A/hI+vrVy1vree/vgPnWQj91jqMAdPrVKG6XVZYoZ2ZPJ+YHg7varl8sVlqdvNZkuZ1KMMZIA57VzySbs9xok0/ZCstvdQsuCRtk6qvbBrQWxKwFldjC7fOCfnVewqnA1trLkytiZMojdCh9SO/NT28k8pksN+LhfldsHGPUf0rCd73+81iaBitksnOPlRSWUnhlxVKxaJnCR2hjTqrO2SfpT2tJlb7MzLJCQCW3YJGf8aa8aWDhZWdl6RcYI+prJbWuakrNiCZdwfa3y4+g/wDr1EkYI5/OkjCG1OxtoLDrT9siDBU49ua9fApaNGM77DvIHrxSiFD1wKZ5rZ9KUEsc170DFj/JGeBSi3p6c8ZqYLnoea3RJWNsMcUwwe1XQhPUU7y+MVYGeYcUhirQ8oDjrSGIdSKYGb5dJ5RrQMIPQU3yPaqAykjPpU6R1l29zOhAT5h/d61s28jSLl49n1NNO4CqpHQ08zGPgcmplVT3psiLTsIbHdIxw6496tCNSuQAR6g1mSpg8Ypsd3JC3ysfpSGanlD0pjBV6kVB9viuE2Sh4mP/AC0jP9Kgk02WXm3vkm/2WO1v8Kl+QE8kkXr+VVnngHVAfqaqTaZqEWS8Dkf7Jz/KqLiQZDRupHqprCU5LoNIvy3cfRUT8qpl43kG9FC9ziqbS7eM0eflSvXNefi5TlHQ1glcnu1DeXFbhDvPDHt70+5toZWG26jeeDBUyc+Zj2HP6UkMUJieGEM08gyAR04/SqcNtDYXsVzPdr52SPKAzgnjqK8OPrt+JsTBvPuRDdRPEoG5se/Sori2jsZZFeYusg3K5649KtXLPeX0QikChMh3xnI9qy9ctZY9RiWdzJbsu5SOBmrpq7tsZyNEHTbfRZF8tRJtIEnVmPaq1ldtpzjdJkS4IYDkN6fSqul6XaXBkMrtuJIQZ4X3qndLKLuWISGUQpuDIMFe3Pr2rRQjJuN7kGhrerx3LyNEMoRtlYfxN7Cqtiw1eeGzm/dQRfOXHXPQAVGLL9y7yFEeNcqAfvZ9ahluvs9ukcKmK5XBwBwRnk1pGKS5Ybg0dJZG0jtpIZo8BWPzg859aS2ubi2vY5bmMIJlzC56FQeh9D7VmWIE9rJO75kzkN0GRWhLNeajosyx2jTBfmIf5SPcVzyjrZiHyXTx628kUQ8iYgGQdA2Oa2pIUWKO5tpHNxwjlv4/rWNp0M2raaFtisYUhgX7MK0tPiluoj9omEc8ZZQo6bgec1hVVvloaRZZmWW1jFzNIhiYbWIJ+Q1E01rqN0oeSR1UcBBhc/jz+lWbWKa8jIZAsQOJFc8v7AVS02IxFHkCozE+WhOSR3rFWs31Rqi7A0LRSLblWBGACw+U++Kqi4uI5WRAzhTjIU4NTzzWUEx/erHu6j3Hf9azl1GVh8x4J4NdeCS573aImaS3Of8AWwMPcCp0MMnKPj2NZi3b/wB6p470j70aNX0tKatqc7RpCNlORyPapApzmq0N1bN/ejPvyKuo6EcSK1dSVyRytinja3UUmFz1FL8o6EVdgHgKKXYDzTMgdx+dAkIosA4xDsKPJpPN9qPNHrTA5tBs4Xge1So5HrTTG5HCmmiKQ1QF2OUHgnFSk8etU44znk4NW1QbeTzTEV5Bmqrrz3rSdFxxUDL3xUtDKBB7UbmXoSKtNGOuKYYx6VLAE1C5i+5Mw9qe2r3OPnWN/qtRGIdqY0Xqal3AWXUYHH72yjPuKoSzacTuEBQg9hT5oevFZ8sLHPFcde7VmXEn/eeYtzbdeQDnFPu7KVZo5UkhLxsCD94OfbHWqUbSxgx5zGex7VYa2l/s6NxM0pY/u1QZI+mK+dqRcJnQndGhePGbQO2yMR/MvOM//rrOtymqTML5CsbMNik42iq0cTx2ciXAledJVJjbkIvX86YJUmuw5Lquwgdt/wBPWkoct7feTIfYQ20cs+/zwsbsTsU4I7HNJLo9zY28+oROHSVsuO6qff1ratLiA6O0cpHkiPGdwB4//VVW0uWmsxazo/lzZBcnoMfzo9pK7ZNjKvbeC3KusjzRsnC9dvp+FVHhj+zRsvF2W2kdSR3/AAxW42nyaXL9ktla5gnXO6Q8r+fasW3MqXbwsoM7AgZ4xj6/Wt4Surp3GQ6fFK8skUrEKW+6p4BroLLUpo7l7JciUJkuT8uK5mO21CPUGO8tKnLFDnIqzHdOmpSOJCAVG5mHJp1aanfroTY6K0uDpz/ZCWIGXDp3z6+9WLF7i9vLohkiBfKt6ZHSsHQonvdRmubySRguAFJxx9PSukhsltdRk+yN8kxBCHsTXHWSi2upcS3LMdJkiSWT5X4STt+NUY1k/tF4CVBtiWjdTxIjc8ep57U3Uzf7lR7fdCGyCg3dK0LuaKysYVcDzCwWM7eVz3rFaLu2aIpzWUcTPHdReZFIN6S9Gz3z71Sm0+SNRJC5mg6hh1H1FS3LzvcrNNcJKpGFIOFGDyMdj0qJFubWZpbdv3bHJj6r9R6V6OC3tIzkuw1M8c1MrleCaso0F4uWType4Hr/AFpklnJGN33k/vCvbhDsYtiLLz1qZZcdDVbbkcCnAHtW8SS6s7DoxqZZz6ms0PjrUqyVqmwNETE9zThKezGqKy89alDe9UmIti5cd81ILs+gqiGpd4qrgSC4hI+9SfaYecGssI1OCMO2adwNDz4ienNPE47CqSIT2q1FCfwpgTGTd3xTTz3qVIgKlEQ7UxFXy/SnCLParax47VMsakdKVgKItQad9i9av4A6CgjNS4gZUlkDxjNUbizCg5wAOSa1r6+t7NP3jZbsg5JrltR1Ca+yp/dxdkHf61hU5UtSkmV5prWRinmYj5yw74qKUyCNYbW4KKRwu7H41UWFmc47dfap7aJ5bwY+Zdp3c8189iYWnzNnTF6WHWt3BZwtbvJvn5J2Atkn1Nal49tdaKN6JuhT5Gz0z1Hsax4IBaXztIuUYbd392kupLaK4SRJVZUO5152nH9a5nFSkmhstyWMFxpyQwnDMR5f/wBeoL++utPgt4ZIkw7ArKG+X3xVgSpNJBLYoZTgsQPlO08d6ytZvTq9qbeFNkdq4355PPAIPpnj64rSlFylaW35EMty31ydUTYyOHUDviMU24kuBDJbG0mdo23m5Vcqozyc9/oKct5ZyaD5EETmXAwEUhg44zmtXSdSh/stLedlLbPLkXqzk55x1NKT5FdR2AyIMLcuYZwUCgs55JaibRp49MM7zI4yGYAc8ntV2Hw1DFpzm2u9kj/vA7DgL6GmWp1P+yWHA2pmNxxkgcAj86PaXd4PqFjVleCLT2n2YaNRtK8MQKp6XqM020mPczyEluwH8+P6UzTFuDp9159s8t42XiMg4ZSMEfzqTSrhNTsrqyMUdtK6HDdAD/jWDgkmnrqWhZXv4tUMrMwRslGU5GPatO5gklaK4wJjt+cDnI6jHrWZps88McNnKMCJXRyeQw7Y/WpVvbqynIIV4WO5UIxtB7D6UKnzTUU7D5rIsX1vFqESmJlSc/dbOAf9k+lYkTy20xjdWRgcMp4wa12tbbUpfPspzbXf8SP91/qP6ipJrZpcRX0RhnAwsvUH8e4r2sNhfZxtuYylcpiXOD1PrV63vcEBzj3qhJDLaNslXGfusOhpEkB4rtjeJD1NkxRycj5Se46GoXt2U5x+VVIrhounK+ladtcLKOPxU9RXRFpk2KZiI5IyKaU9K1WgBGV6GoXg9sVaQijgqc1IrVIY8cVGVwaYDw2aN3tUfIOMUZx1FMColwelTLcetY8EzMil12MRyKtI/vVJjNVJ6sJcZ4BrIV8fxVMk2OlO4jYS4Gcc5qdJWx/jWRHcE9MCrUcjN1ancDSDnuRUitVISKg3OwAHcnFQTauqjbANx/vEcUm0hWNZ5khj3yMFHqayLzV5HBW2XaP75/pVCSaS5cF2Lv2H/wBanraTN94rEPVzj9Kzcm9h2KMgLOWclmPJJ5NMispbs/IpCeta6WllEMyFrh/TotLPMzx7eEjHRF4FZOn3Hcx5beCBcMd2OiL0/E96zri6k6Q/JngBa0bqMYJNUU3JcBYY98jDCrjrXFXg+iNIsqbbrbHGznjgA9s9/rVu6sopIo7bcFt4z8zL1Y+/tV2a0EUCNcYeQDkZ4qqLh50MBUfLli/oo7f0ryqlKpFq+5opJk8NjLqdvLcRwxRAjYrg4woHU+grHmt1tdFuJIiRbtIPNbPMuD8oX2J5/Crbx3Oo6WzjfHHI5IROFYD19aFtfK0R/MkEqRfejJ/iPQD/AD61nB8ujfXYe5HGFn8PGHTS0JmfLGQ8r0OM1c02EmTUXhnie7eMJFt5wcYYj86ppNqEcuLe2BtLlVBQj7uM9PSrSSHS7iK8MLbZplXy2Od2Rg49MD9cU5t2aXXX/hwSGH7Tp2gW0UluxYytG28cBc5x+PrWxFdb7dmtoZjE6kR5GSB0/Qg1nWLtNrV2ZhvhY/JC/IxngAU2Rv7S+aKV7ezAbyVX+IBiCD7ZH61lOKk7P1v6jSJbfUdUknaARIG4UkKScDv+NXLCG2mvJpEjLZdmDdmGc7x7+vvVH+0jZazCiZaBIkDKRyD3x+OKtIrmUgYJD70QDgxtnHPfP9KicdNrXGMuruKe/jmRCPLYDPTI71PehhJluR2rLinxNIjLlCxwO4rbhxc2a5O5lG0n+tehhsLrcznIz1GDuGQ3Wtez1QhPJuR5kXTJ61mvGUbHSmjj8K9SneBk9TpPscFzbkQuHjP8BPT8e1Yl5pz2bFgCY/UjkfWi3uXhbKMVP1rUi1NpF2yBXHQ5711WjJE6owgeeePrT1ZlYEZBHStZ7fT5udjQk/3Dx+VQPpqE/u7kH03Cj2bWwXLNleiQhJCFY/ka0Gi4rDNjMvRkb6HFalhcvgW9wCrjhWPetI32YmDwioHgz0rTePNQMuKuwjKeEjmm+Wa0mjB7VEYOaLAciBTwxA60pX0oCVBYqyN3qQSEdqjwT7U4KR0paiJ0uGHQVMLyY8LxVUDHWpFzjHXPQDvRdgSmRmO6Qlj6ntVy2s2lAd/kQ9B3NPtLEIFknAz2X0q8TnhapR6sVyEQiJSEG0d/ekEZ/CpxGep6UpPbFOwiuI6jljwOetWSCBmq0uM5zxUtDKM0TSuscalmY4AFaVvYRWEJPDSsPmf/AD2qzY2/lp50g+dhx7CmXj449qhxS1C5g6g5lc/3RU2n6YptHkuB8snUHuOwqWO0Nzcqh+71b6VpXK7k2Lwo4Fc6pJtzY79DnL28nEj+S+yMcAYqsqg2gnmJkVW3yKeuR0rQvbUD5QKqTxFNKmI/vL/hXmYjDato1jIhsZDc2s0wfypSSsag446ULp8MtzBh2PkoN3s3p/WmwxsbSJVGGLFVA75rXMH2BYY4xnAO73NcyoVHzOJfMjOa+hkQpLGY9pZS6KeG7HNPsw9ksdtKo+yWzM5P94MD/ImoJ4AZSSOSckGtJIftGnvGRmRUIX8ulNYZuNkg5tSOaWOe6ilt412ttQkjtmogkmnak7x/xZHPpUGmyAEQsTw4Zfz6V02pWAY7wPxrqw2D91qREp6nLSoRLkjk81oadP5MoVj8j8GmXMJB5FQbSGx2rtjHkehO5vXNvuGQOaznQqeRWjpt2J0+zSn94B8p/vCn3FqRniuvlT1RBkinq5HtTniKE8UzGDQlYLlpZg3B4NSiXFUgTUqscVohFoSZ6GpEmI4PI9KqA5pwY1SYG/DOJUAPWnOtZNvcGNhnpWj52UDKMjvVCGOtR/jUxkVqaVGaAOU8s96UR+lTDHfmjAzUjItgHWkK1IRTcjtSGN2/hWpY2ohAlkHznoD2qCxtwzea3IXpn1rRxk04oQ8ksetSLGKRIwe9TFcAAVYhp4HFAJHbmpQnA9ajkzyAaTArvhs+lMihEtwoxlRyaGHBPartpEIodzfebk1G4EsjhEz7cVkykuxY9e1Xbh9zEdhTLWIPJvP3V6fWolq7DC2hMMOMfvG+8fT2qb7P/e5qdU4J6AcCpjHhQcU7CMO9h3MTj2rLvYtunMv96RR/M/0robtADjHasfUF/dwJ/tFz/L/GuepHdlIj0Wx82VXYfLDkj6mtOa2Lv0qzodvt0/eRy7E/0q48YBAxzVwpJQSBvU5O8tSr9Kdakoy1sX1sOpFZJTyx+NT7NJhczr60MF84TgN86H612cDC+0+Gbj50BP17/rXP3sXnWaTj70R2t/unp+tX/DlySklqT935l+h61UI2k13B7Fe+ttkmMcVjSxlJOK7O8tw65xmudurYhjjtVSgCZQjchgQcMDkH3ro7O5W+t/mx5q/eHr71ze3axJFWLad7aVZU7dR6iiGjBmvPb5BwOaz5IipzitsMlxCJU6NVaaAOpwMGtrEmTjnpSqKlkRgQccios880rDJR0oBpEp7DPIpgOU81agnaM8niqPSnq5HemgNcbW+YdDUm32rNhmKnrxVoS8fepgYGQBTS5zxUXmUbsnrSAk6n1qSOMuwUVEpArRtY9ibm+81CVwJ0QKqqowBxVhIwelMQAnHep1wBgVQiRUxxT9nPJpiMfTJpxbPTigBXGBwaquCRkmpJGO04NVmY49qhgPiTzZVU9Op+lXZZAM1UtPlV3IxuOB9KJZAX46Ck9EBHIST3rQt4wkap36tVS2jMjbyOAePetSKPb1/GiMeoMcibm56CpXXI9qcigCkm4iNNrQDKusFyfwrJvQDPjsigVryYZxnp1rJ5mlyesj5/WsZrSw0dLYw+XYQr6IKc6j0qaMbVC+gpjjn8a3toIpTx74mB61h3URDV00iA8isq8h+UsB14qXEDNs2RmaCU/u5QUb2z3/CqVpI+n34Zlw0TlJB7dDVhl8t8Gm36b9lwOS/yv7kDr+VZtdSkdaCs0QKkFSMg1kXdttdsUnh673I1q55TlPpWpcxZG/HNarVXJOTnt9ueOtVNpXiuimgDBhisua1OCR2qXEdxdNvTbS7HP7p+vsfWtiYY5HQ965sqy9a1dNvRIn2aU8j7pP8AKqi+giSRBIMj7wqm6ZJBGDVyVdrHtULFXODwabGVFJXqKnVtw5pjrtYhuaFyOnSkBIycVHjHFSqcikZc9KYDQeafvPrUJ4NLvNFwME3IHemG9iTlpAPxrg/7QnbrNIf+BGpraR5ZQoyWY4FZ8wHoWnTx3cp2HKJyT2rdjOTWJpUC2tskI6jlj6mtpGAFaoCynB4HNSphScjJqBHwcipN/HuaYibzMdOtG4EYqEMCPencAHFIBJXAXHaoDlwOwpxy30pBjIHYVLGSb9sfHbpUagyuFHUmo5JMtgdBVqyjyTIe/A+lK12BegQAAAcAcVaUHNMRMKKnXGMitCSUDAqG5bjFTZGCap3DDGaljKFw2I5Gz2x+dUrRN17AuONwqzcnMeP7xqPThnUY/QAn/P51k1eSGjpM4HvUTHrTlPHNROeSexrckc/K8duaguIg8RHerIGUP0pjr8ucdKAOcuYjux3zUGzzIJIT1IyPqK1b2Hdll6is4fLKDjvUNDRSs7g2t3FMP4WGfp3rtjiSMHqCK4adNk7pjjJxXV6Ncefp6AnLINppQ7DYyaHDHaao+ScsOtbE6dGFUnXkkVpYkyprXcM4rPeF4XyMjB4I7V0YUEGopLVXBBHWpcRmfDdi4jCScSAfnUchwxp8thhuOMVE0Mu3ruxQMBJng80dDkdKrsxQ8jFOWQ9c0XAtA57Yp3eoVcHvUm71oAR1zUW2pjg0m2lYDxFQc1saGmdQiz25rPEfNa+iri9U+xrGO4Hd2bYArSjO41j2rHArSSTA4rdMC8JNowKcCTjmqqNuFTqcDmmBMCQODQWLcA8UzOeO1Ix7CgALYXimF9qZNVp5xvWFD8xPzH0FEkmSBUtgSpmSRVHVjW7bxhVAHQcVj6aheZpP7vArbT5acUJkwFTj2qBOmTUobkYqhDyflNUJznirjHGRVC4btSYylcHlR7Zo00/6cfZD/MUydssx7dKTTmH21vdD/MVmviGdErfJULHAIzxmlDfKKimPHHrWxJaTlCO9SDBGCO1QxN8n4VMBwPWgDPuo8ZI6VkSRgOCOhreuANprGuEKscdM8UhmZqCbLkH+8oNaHh6fbO8RPDDIqnqfIhbvgjNRafN5F9C3bdistpDOycZ47VQkADlfyrQPIyOtVLlAQHHUVsSVhgZp+3IBFRZwwJ6Gp044HQ0DIZo8jP51QkXYT7VrlQciqVxH19aGBlzqMhu3Q1WeLbyvFXJB1WoFPGKzkNFfJU09Zh3pzpmoHSlcZZ81fWk89fWqmD2pMUcwWPMQvNXrBjHcRkddwFUhVqDggjrWC3A7W2YgYrRjY4rMtTmNT32itJOgreIFtGx0qdSe/Sq0fT8amXk1YibJPPaqVzfAApFye7Ut8zARqCQrZyB3rO6jmonK2g0iS3bMpYntUzP3qCH+OpO4qUNnQadH5dqmep5NaAJ4qvB9xfpU461siCdTlQTUhOce1Rp92pOxoAbI2Kz5z8w/OtB+1Ztx95vpSYFKU5TNRWLldQUeqGpJfun6VBZf8hNf90/yrNbjOkV8pmoJXOCD1Bp0X+pqK4+6a2EXbdsoAatRnjPeqNt91aux0IRDMODWTcja49DWvN1rJuvvGgZl34zAmezGs8NtYEdjWhf/AOpH1/xrPHU1jLcpHbW0omtY3B4YCkccFT0NVNGJOmLn3q3J0rZbEmbJmOQoenUVLBJztqO+6p9aZGTvBpAaQ+YfSoJUzk1Knf6UjfcNUBiXSFTVHcUc+hrTvvu/hWY4+UVEhkoIYUx0psR+apm6VBRUZcUYFSuKZgUAf//Z" + }, + "key":"1" + } + ] + } +kind: ConfigMap +metadata: + annotations: + name: tf-vegeta-cfg