Add proxy/dequeuer CPU and memory requests

docs/workloads/async/containers.md

@@ -27,6 +27,10 @@ Your API pod can contain multiple containers, only one of which can be listening
 
 The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers.
 
+## Resource requests
+
+Each container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's dequeuer sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory.
+
 ## Observability
 
 See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md).

docs/workloads/batch/containers.md

@@ -33,6 +33,10 @@ Your API pod can contain multiple containers, only one of which can be listening
 
 The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers.
 
+## Resource requests
+
+Each container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's dequeuer sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory.
+
 ## Observability
 
 See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md).

docs/workloads/realtime/containers.md

@@ -25,6 +25,10 @@ Your API pod can contain multiple containers, only one of which can be listening
 
 The `/mnt` directory is mounted to each container's file system, and is shared across all containers.
 
+## Resource requests
+
+Each container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's proxy sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory.
+
 ## Observability
 
 See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md).

pkg/config/config.go

@@ -40,8 +40,7 @@ import (
 var (
 	OperatorMetadata *clusterconfig.OperatorMetadata
 
-	ClusterConfig     *clusterconfig.Config
-	InstancesMetadata []aws.InstanceMetadata
+	ClusterConfig *clusterconfig.Config
 
 	AWS             *aws.Client
 	K8s             *k8s.Client
@@ -92,10 +91,6 @@ func Init() error {
 
 	ClusterConfig = clusterConfig
 
-	for _, instanceType := range clusterConfig.GetAllInstanceTypes() {
-		InstancesMetadata = append(InstancesMetadata, aws.InstanceMetadatas[clusterConfig.Region][instanceType])
-	}
-
 	AWS, err = aws.NewForRegion(clusterConfig.Region)
 	if err != nil {
 		return err

pkg/consts/consts.go

@@ -18,6 +18,8 @@ package consts
 
 import (
 	"os"
+
+	kresource "k8s.io/apimachinery/pkg/api/resource"
 )
 
 var (
@@ -38,6 +40,11 @@ var (
 
 	StatsDPortStr = "9125"
 
+	CortexProxyCPU    = kresource.MustParse("100m")
+	CortexProxyMem    = kresource.MustParse("100Mi")
+	CortexDequeuerCPU = kresource.MustParse("100m")
+	CortexDequeuerMem = kresource.MustParse("100Mi")
+
 	AuthHeader = "X-Cortex-Authorization"
 
 	DefaultInClusterConfigPath   = "/configs/cluster/cluster.yaml"

pkg/lib/k8s/quantity.go

@@ -95,6 +95,22 @@ func NewMilliQuantity(milliValue int64) *Quantity {
 	}
 }
 
+// Returns nil if no quantities are passed in
+func NewSummed(quantities ...kresource.Quantity) *Quantity {
+	if len(quantities) == 0 {
+		return nil
+	}
+
+	k8sQuantity := kresource.Quantity{}
+	for _, q := range quantities {
+		k8sQuantity.Add(q)
+	}
+
+	return &Quantity{
+		Quantity: k8sQuantity,
+	}
+}
+
 func (quantity *Quantity) MilliString() string {
 	return s.Int64(quantity.Quantity.MilliValue()) + "m"
 }
@@ -103,10 +119,89 @@ func (quantity *Quantity) ToFloat32() float32 {
 	return float32(quantity.Quantity.MilliValue()) / float32(1000)
 }
 
-func (quantity *Quantity) ToKi() int64 {
-	kiFloat := float64(quantity.Quantity.Value()) / float64(1024)
+func ToKiRounded(k8sQuantity kresource.Quantity) int64 {
+	kiFloat := float64(k8sQuantity.Value()) / float64(1024)
 	return int64(math.Round(kiFloat))
 }
+func ToKiRoundedStr(k8sQuantity kresource.Quantity) string {
+	return s.Int64(ToKiRounded(k8sQuantity)) + "Ki"
+}
+func (quantity *Quantity) ToKiRounded() int64 {
+	return ToKiRounded(quantity.Quantity)
+}
+func (quantity *Quantity) ToKiRoundedStr() string {
+	return ToKiRoundedStr(quantity.Quantity)
+}
+
+func ToKiCeil(k8sQuantity kresource.Quantity) int64 {
+	kiFloat := float64(k8sQuantity.Value()) / float64(1024)
+	return int64(math.Ceil(kiFloat))
+}
+func ToKiCeilStr(k8sQuantity kresource.Quantity) string {
+	return s.Int64(ToKiCeil(k8sQuantity)) + "Ki"
+}
+func (quantity *Quantity) ToKiCeil() int64 {
+	return ToKiCeil(quantity.Quantity)
+}
+func (quantity *Quantity) ToKiCeilStr() string {
+	return ToKiCeilStr(quantity.Quantity)
+}
+
+func ToKiFloor(k8sQuantity kresource.Quantity) int64 {
+	kiFloat := float64(k8sQuantity.Value()) / float64(1024)
+	return int64(math.Floor(kiFloat))
+}
+func ToKiFloorStr(k8sQuantity kresource.Quantity) string {
+	return s.Int64(ToKiFloor(k8sQuantity)) + "Ki"
+}
+func (quantity *Quantity) ToKiFloor() int64 {
+	return ToKiFloor(quantity.Quantity)
+}
+func (quantity *Quantity) ToKiFloorStr() string {
+	return ToKiFloorStr(quantity.Quantity)
+}
+
+func ToMiRounded(k8sQuantity kresource.Quantity) int64 {
+	miFloat := float64(k8sQuantity.Value()) / float64(1024*1024)
+	return int64(math.Round(miFloat))
+}
+func ToMiRoundedStr(k8sQuantity kresource.Quantity) string {
+	return s.Int64(ToMiRounded(k8sQuantity)) + "Mi"
+}
+func (quantity *Quantity) ToMiRounded() int64 {
+	return ToMiRounded(quantity.Quantity)
+}
+func (quantity *Quantity) ToMiRoundedStr() string {
+	return ToMiRoundedStr(quantity.Quantity)
+}
+
+func ToMiCeil(k8sQuantity kresource.Quantity) int64 {
+	miFloat := float64(k8sQuantity.Value()) / float64(1024*1024)
+	return int64(math.Ceil(miFloat))
+}
+func ToMiCeilStr(k8sQuantity kresource.Quantity) string {
+	return s.Int64(ToMiCeil(k8sQuantity)) + "Mi"
+}
+func (quantity *Quantity) ToMiCeil() int64 {
+	return ToMiCeil(quantity.Quantity)
+}
+func (quantity *Quantity) ToMiCeilStr() string {
+	return ToMiCeilStr(quantity.Quantity)
+}
+
+func ToMiFloor(k8sQuantity kresource.Quantity) int64 {
+	miFloat := float64(k8sQuantity.Value()) / float64(1024*1024)
+	return int64(math.Floor(miFloat))
+}
+func ToMiFloorStr(k8sQuantity kresource.Quantity) string {
+	return s.Int64(ToMiFloor(k8sQuantity)) + "Mi"
+}
+func (quantity *Quantity) ToMiFloor() int64 {
+	return ToMiFloor(quantity.Quantity)
+}
+func (quantity *Quantity) ToMiFloorStr() string {
+	return ToMiFloorStr(quantity.Quantity)
+}
 
 // SplitInTwo divides the quantity in two and return both halves (ensuring they add up to the original value)
 func (quantity *Quantity) SplitInTwo() (*kresource.Quantity, *kresource.Quantity) {
@@ -169,6 +264,13 @@ func (quantity *Quantity) ID() string {
 	return s.Int64(quantity.MilliValue())
 }
 
+func (quantity *Quantity) DeepCopy() Quantity {
+	return Quantity{
+		Quantity:   quantity.Quantity.DeepCopy(),
+		UserString: quantity.UserString,
+	}
+}
+
 func QuantityPtr(k8sQuantity kresource.Quantity) *kresource.Quantity {
 	return &k8sQuantity
 }

pkg/operator/endpoints/info.go

@@ -38,9 +38,8 @@ func Info(w http.ResponseWriter, r *http.Request) {
 	}
 
 	fullClusterConfig := clusterconfig.InternalConfig{
-		Config:            *config.ClusterConfig,
-		OperatorMetadata:  *config.OperatorMetadata,
-		InstancesMetadata: config.InstancesMetadata,
+		Config:           *config.ClusterConfig,
+		OperatorMetadata: *config.OperatorMetadata,
 	}
 
 	response := schema.InfoResponse{

pkg/operator/operator/memory_capacity.go

@@ -18,6 +18,7 @@ package operator
 
 import (
 	"github.com/cortexlabs/cortex/pkg/config"
+	"github.com/cortexlabs/cortex/pkg/lib/aws"
 	"github.com/cortexlabs/cortex/pkg/lib/k8s"
 	"github.com/cortexlabs/cortex/pkg/lib/slices"
 	kresource "k8s.io/apimachinery/pkg/api/resource"
@@ -104,9 +105,10 @@ func UpdateMemoryCapacityConfigMap() (map[string]kresource.Quantity, error) {
 	primaryInstances := []string{}
 
 	minMemMap := map[string]kresource.Quantity{}
-	for _, instanceMetadata := range config.InstancesMetadata {
-		minMemMap[instanceMetadata.Type] = instanceMetadata.Memory
-		primaryInstances = append(primaryInstances, instanceMetadata.Type)
+	for _, ng := range config.ClusterConfig.NodeGroups {
+		instanceMetadata := aws.InstanceMetadatas[config.ClusterConfig.Region][ng.InstanceType]
+		minMemMap[ng.InstanceType] = instanceMetadata.Memory
+		primaryInstances = append(primaryInstances, ng.InstanceType)
 	}
 
 	nodeMemCapacityMap, err := getMemoryCapacityFromNodes(primaryInstances)

pkg/operator/resources/errors.go

@@ -73,14 +73,10 @@ func ErrorCannotChangeKindOfDeployedAPI(name string, newKind, prevKind userconfi
 	})
 }
 
-func ErrorNoAvailableNodeComputeLimit(resource string, reqStr string, maxStr string) error {
-	message := fmt.Sprintf("no instances can satisfy the requested %s quantity - requested %s %s but instances only have %s %s available", resource, reqStr, resource, maxStr, resource)
-	if maxStr == "0" {
-		message = fmt.Sprintf("no instances can satisfy the requested %s quantity - requested %s %s but instances don't have any %s", resource, reqStr, resource, resource)
-	}
+func ErrorNoAvailableNodeComputeLimit(msg string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrNoAvailableNodeComputeLimit,
-		Message: message,
+		Message: msg,
 	})
 }
 
@@ -105,6 +101,6 @@ func ErrorAPIsNotDeployed(notDeployedAPIs []string) error {
 func ErrorInvalidNodeGroupSelector(selected string, availableNodeGroups []string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrInvalidNodeGroupSelector,
-		Message: fmt.Sprintf("node group %s doesn't exist; remove the node group selector to let Cortex determine automatically where to place the API or specify a valid node group name (%s)", selected, s.StrsOr(availableNodeGroups)),
+		Message: fmt.Sprintf("node group \"%s\" doesn't exist; remove the node group selector to let Cortex determine automatically where to place the API, or specify a valid node group name (%s)", selected, s.StrsOr(availableNodeGroups)),
 	})
 }