Add replica autoscaling to GCP clusters

CONTRIBUTING.md

@@ -213,6 +213,7 @@ max_instances: 5
 image_operator: /cortexlabs/operator:latest
 image_manager: gcr.io/<project_id>/cortexlabs/manager:latest
 image_downloader: gcr.io/<project_id>/cortexlabs/downloader:latest
+image_request_monitor: gcr.io/<project_id>/cortexlabs/request-monitor:latest
 image_istio_proxy: gcr.io/<project_id>/cortexlabs/istio-proxy:latest
 image_istio_pilot: gcr.io/<project_id>/cortexlabs/istio-pilot:latest
 image_google_pause: gcr.io/<project_id>/cortexlabs/google-pause:latest

docs/clusters/gcp/install.md

@@ -74,6 +74,7 @@ The docker images used by the Cortex cluster can also be overridden, although th
 image_operator: quay.io/cortexlabs/operator:master
 image_manager: quay.io/cortexlabs/manager:master
 image_downloader: quay.io/cortexlabs/downloader:master
+image_request_monitor: quay.io/cortexlabs/request-monitor:master
 image_istio_proxy: quay.io/cortexlabs/istio-proxy:master
 image_istio_pilot: quay.io/cortexlabs/istio-pilot:master
 image_google_pause: quay.io/cortexlabs/google-pause:master

pkg/operator/operator/k8s.go

@@ -930,9 +930,16 @@ func neuronRuntimeDaemonContainer(api *spec.API, volumeMounts []kcore.VolumeMoun
 }
 
 func RequestMonitorContainer(api *spec.API) kcore.Container {
+	var image string
+	if config.Provider == types.AWSProviderType {
+		image = config.CoreConfig.ImageRequestMonitor
+	} else if config.Provider == types.GCPProviderType {
+		image = config.GCPCoreConfig.ImageRequestMonitor
+	}
+
 	return kcore.Container{
 		Name:            _requestMonitorContainerName,
-		Image:           config.CoreConfig.ImageRequestMonitor,
+		Image:           image,
 		ImagePullPolicy: kcore.PullAlways,
 		Args:            []string{"-p", DefaultRequestMonitorPortStr},
 		Ports: []kcore.ContainerPort{

pkg/operator/resources/realtimeapi/api.go

@@ -29,7 +29,6 @@ import (
 	"github.com/cortexlabs/cortex/pkg/operator/lib/routines"
 	"github.com/cortexlabs/cortex/pkg/operator/operator"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
-	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/cortexlabs/cortex/pkg/types/spec"
 	"github.com/cortexlabs/cortex/pkg/types/status"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
@@ -325,10 +324,8 @@ func applyK8sDeployment(api *spec.API, prevDeployment *kapps.Deployment) error {
 		}
 	}
 
-	if config.Provider == types.AWSProviderType {
-		if err := UpdateAutoscalerCron(newDeployment, api); err != nil {
-			return err
-		}
+	if err := UpdateAutoscalerCron(newDeployment, api); err != nil {
+		return err
 	}
 
 	return nil

pkg/operator/resources/realtimeapi/k8s_specs.go

@@ -19,9 +19,7 @@ package realtimeapi
 import (
 	"github.com/cortexlabs/cortex/pkg/lib/k8s"
 	"github.com/cortexlabs/cortex/pkg/lib/pointer"
-	"github.com/cortexlabs/cortex/pkg/operator/config"
 	"github.com/cortexlabs/cortex/pkg/operator/operator"
-	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/cortexlabs/cortex/pkg/types/spec"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
 	istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1"
@@ -45,12 +43,8 @@ func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Depl
 }
 
 func tensorflowAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment {
-
 	containers, volumes := operator.TensorFlowPredictorContainers(api)
-
-	if config.Provider == types.AWSProviderType {
-		containers = append(containers, operator.RequestMonitorContainer(api))
-	}
+	containers = append(containers, operator.RequestMonitorContainer(api))
 
 	return k8s.Deployment(&k8s.DeploymentSpec{
 		Name:           operator.K8sName(api.Name),
@@ -98,10 +92,7 @@ func tensorflowAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.D
 
 func pythonAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment {
 	containers, volumes := operator.PythonPredictorContainers(api)
-
-	if config.Provider == types.AWSProviderType {
-		containers = append(containers, operator.RequestMonitorContainer(api))
-	}
+	containers = append(containers, operator.RequestMonitorContainer(api))
 
 	return k8s.Deployment(&k8s.DeploymentSpec{
 		Name:           operator.K8sName(api.Name),
@@ -149,10 +140,7 @@ func pythonAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deplo
 
 func onnxAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment {
 	containers, volumes := operator.ONNXPredictorContainers(api)
-
-	if config.Provider == types.AWSProviderType {
-		containers = append(containers, operator.RequestMonitorContainer(api))
-	}
+	containers = append(containers, operator.RequestMonitorContainer(api))
 
 	return k8s.Deployment(&k8s.DeploymentSpec{
 		Name:           operator.K8sName(api.Name),

pkg/types/clusterconfig/cluster_config_gcp.go

@@ -47,6 +47,7 @@ type GCPCoreConfig struct {
 	ImageOperator                 string `json:"image_operator" yaml:"image_operator"`
 	ImageManager                  string `json:"image_manager" yaml:"image_manager"`
 	ImageDownloader               string `json:"image_downloader" yaml:"image_downloader"`
+	ImageRequestMonitor           string `json:"image_request_monitor" yaml:"image_request_monitor"`
 	ImageClusterAutoscaler        string `json:"image_cluster_autoscaler" yaml:"image_cluster_autoscaler"`
 	ImageFluentBit                string `json:"image_fluent_bit" yaml:"image_fluent_bit"`
 	ImageIstioProxy               string `json:"image_istio_proxy" yaml:"image_istio_proxy"`
@@ -167,6 +168,13 @@ var GCPCoreConfigStructFieldValidations = []*cr.StructFieldValidation{
 			Validator: validateImageVersion,
 		},
 	},
+	{
+		StructField: "ImageRequestMonitor",
+		StringValidation: &cr.StringValidation{
+			Default:   "quay.io/cortexlabs/request-monitor:" + consts.CortexVersion,
+			Validator: validateImageVersion,
+		},
+	},
 	{
 		StructField: "ImageClusterAutoscaler",
 		StringValidation: &cr.StringValidation{
@@ -655,6 +663,7 @@ func (cc *GCPCoreConfig) UserTable() table.KeyValuePairs {
 	items.Add(ImageOperatorUserKey, cc.ImageOperator)
 	items.Add(ImageManagerUserKey, cc.ImageManager)
 	items.Add(ImageDownloaderUserKey, cc.ImageDownloader)
+	items.Add(ImageRequestMonitorUserKey, cc.ImageRequestMonitor)
 	items.Add(ImageClusterAutoscalerUserKey, cc.ImageClusterAutoscaler)
 	items.Add(ImageFluentBitUserKey, cc.ImageFluentBit)
 	items.Add(ImageIstioProxyUserKey, cc.ImageIstioProxy)
@@ -739,6 +748,9 @@ func (cc *GCPCoreConfig) TelemetryEvent() map[string]interface{} {
 	if !strings.HasPrefix(cc.ImageDownloader, "cortexlabs/") {
 		event["image_downloader._is_custom"] = true
 	}
+	if !strings.HasPrefix(cc.ImageRequestMonitor, "cortexlabs/") {
+		event["image_request_monitor._is_custom"] = true
+	}
 	if !strings.HasPrefix(cc.ImageClusterAutoscaler, "cortexlabs/") {
 		event["image_cluster_autoscaler._is_custom"] = true
 	}