Prometheus (#1814)

Author: Miguel Varela Ramos

CONTRIBUTING.md

@@ -187,9 +187,13 @@ image_inferentia: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/inferen
 image_neuron_rtd: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/neuron-rtd:latest
 image_nvidia: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/nvidia:latest
 image_fluent_bit: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/fluent-bit:latest
-image_statsd: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/statsd:latest
 image_istio_proxy: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/istio-proxy:latest
 image_istio_pilot: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/istio-pilot:latest
+image_prometheus: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/prometheus:latest
+image_prometheus_config_reloader: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/prometheus-config-reloader:latest
+image_prometheus_operator: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/prometheus-operator:latest
+image_prometheus_statsd_exporter: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/prometheus-statsd-exporter:latest
+image_prometheus_to_cloudwatch: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/prometheus-to-cloudwatch:latest
 ```
 
 Create `dev/config/cluster-gcp.yaml`. Paste the following config, and update `project`, `zone`, and all registry URLs (replace `<project_id>` with your project ID, and update `gcr.io` if you are using a different host):
@@ -212,6 +216,11 @@ image_downloader: gcr.io/<project_id>/cortexlabs/downloader:latest
 image_istio_proxy: gcr.io/<project_id>/cortexlabs/istio-proxy:latest
 image_istio_pilot: gcr.io/<project_id>/cortexlabs/istio-pilot:latest
 image_google_pause: gcr.io/<project_id>/cortexlabs/google-pause:latest
+image_prometheus: gcr.io/<project_id>/cortexlabs/prometheus:latest
+image_prometheus_config_reloader: gcr.io/<project_id>/cortexlabs/prometheus-config-reloader:latest
+image_prometheus_operator: gcr.io/<project_id>/cortexlabs/prometheus-operator:latest
+image_prometheus_statsd_exporter: gcr.io/<project_id>/cortexlabs/prometheus-statsd-exporter:latest
+image_prometheus_stackdriver_sidecar: gcr.io/<project_id>/cortexlabs/prometheus-stackdriver-sidecar:latest
 ```
 
 ### Building

build/images.sh

@@ -52,10 +52,10 @@ api_slim_images_gcp=(
 dev_images_cluster=(
   "downloader"
   "manager"
+  "request-monitor"
 )
 dev_images_aws=(
   # includes dev_images_cluster
-  "request-monitor"
 )
 dev_images_gcp=(
   # includes dev_images_cluster
@@ -69,6 +69,10 @@ non_dev_images_cluster=(
   "istio-proxy"
   "istio-pilot"
   "fluent-bit"
+  "prometheus"
+  "prometheus-config-reloader"
+  "prometheus-operator"
+  "prometheus-statsd-exporter"
 )
 non_dev_images_aws=(
   # includes non_dev_images_cluster
@@ -77,11 +81,12 @@ non_dev_images_aws=(
   "inferentia"
   "neuron-rtd"
   "nvidia"
-  "statsd"
+  "prometheus-to-cloudwatch"
 )
 non_dev_images_gcp=(
   # includes non_dev_images_cluster
   "google-pause"
+  "prometheus-stackdriver-sidecar"
 )
 
 all_images=(

cli/cmd/cluster.go

@@ -208,10 +208,10 @@ var _clusterUpCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		err = createOrClearDashboard(awsClient, clusterConfig.ClusterName)
-		if err != nil {
-			exit.Error(err)
-		}
+		//err = createOrClearDashboard(awsClient, clusterConfig.ClusterName)
+		//if err != nil {
+		//	exit.Error(err)
+		//}
 
 		out, exitCode, err := runManagerWithClusterConfig("/root/install.sh", clusterConfig, awsCreds, nil, nil)
 		if err != nil {
@@ -483,15 +483,15 @@ var _clusterDownCmd = &cobra.Command{
 			prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted, are you sure you want to continue?", *accessConfig.ClusterName, *accessConfig.Region), "", "")
 		}
 
-		fmt.Print("￮ deleting dashboard ")
-		err = awsClient.DeleteDashboard(*accessConfig.ClusterName)
-		if err != nil {
-			fmt.Printf("\n\nunable to delete cortex's api dashboard (see error below); if it still exists after the cluster has been deleted, please delete it via the cloudwatch console: https://%s.console.aws.amazon.com/cloudwatch/home#dashboards:\n", *accessConfig.Region)
-			errors.PrintError(err)
-			fmt.Println()
-		} else {
-			fmt.Println("✓")
-		}
+		//fmt.Print("￮ deleting dashboard ")
+		//err = awsClient.DeleteDashboard(*accessConfig.ClusterName)
+		//if err != nil {
+		//	fmt.Printf("\n\nunable to delete cortex's api dashboard (see error below); if it still exists after the cluster has been deleted, please delete it via the cloudwatch console: https://%s.console.aws.amazon.com/cloudwatch/home#dashboards:\n", *accessConfig.Region)
+		//	errors.PrintError(err)
+		//	fmt.Println()
+		//} else {
+		//	fmt.Println("✓")
+		//}
 
 		fmt.Print("￮ deleting sqs queues ")
 		err = awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(*accessConfig.ClusterName))
@@ -779,6 +779,7 @@ func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterco
 	eksPrice := aws.EKSPrices[*clusterConfig.Region]
 	operatorInstancePrice := aws.InstanceMetadatas[*clusterConfig.Region]["t3.medium"].Price
 	operatorEBSPrice := aws.EBSMetadatas[*clusterConfig.Region]["gp2"].PriceGB * 20 / 30 / 24
+	metricsEBSPrice := aws.EBSMetadatas[*clusterConfig.Region]["gp2"].PriceGB * 40 / 30 / 24
 	nlbPrice := aws.NLBMetadatas[*clusterConfig.Region].Price
 	natUnitPrice := aws.NATMetadatas[*clusterConfig.Region].Price
 	apiEBSPrice := aws.EBSMetadatas[*clusterConfig.Region][clusterConfig.InstanceVolumeType.String()].PriceGB * float64(clusterConfig.InstanceVolumeSize) / 30 / 24
@@ -793,7 +794,8 @@ func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterco
 		natTotalPrice = natUnitPrice * float64(len(clusterConfig.AvailabilityZones))
 	}
 
-	totalPrice := eksPrice + totalAPIInstancePrice + apiEBSPrice*float64(numAPIInstances) + operatorInstancePrice + operatorEBSPrice + nlbPrice*2 + natTotalPrice
+	totalPrice := eksPrice + totalAPIInstancePrice + apiEBSPrice*float64(numAPIInstances) +
+		operatorInstancePrice*2 + operatorEBSPrice + metricsEBSPrice + nlbPrice*2 + natTotalPrice
 	fmt.Printf(console.Bold("\nyour cluster currently costs %s per hour\n\n"), s.DollarsAndCents(totalPrice))
 
 	headers := []table.Header{
@@ -805,8 +807,9 @@ func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterco
 	rows = append(rows, []interface{}{"1 eks cluster", s.DollarsMaxPrecision(eksPrice)})
 	rows = append(rows, []interface{}{fmt.Sprintf("%d %s for your apis", numAPIInstances, s.PluralS("instance", numAPIInstances)), s.DollarsAndTenthsOfCents(totalAPIInstancePrice) + " total"})
 	rows = append(rows, []interface{}{fmt.Sprintf("%d %dgb ebs %s for your apis", numAPIInstances, clusterConfig.InstanceVolumeSize, s.PluralS("volume", numAPIInstances)), s.DollarsAndTenthsOfCents(apiEBSPrice*float64(numAPIInstances)) + " total"})
-	rows = append(rows, []interface{}{"1 t3.medium instance for the operator", s.DollarsMaxPrecision(operatorInstancePrice)})
+	rows = append(rows, []interface{}{"2 t3.medium instances for the cortex system", s.DollarsMaxPrecision(operatorInstancePrice * 2)})
 	rows = append(rows, []interface{}{"1 20gb ebs volume for the operator", s.DollarsAndTenthsOfCents(operatorEBSPrice)})
+	rows = append(rows, []interface{}{"1 40gb ebs volume for prometheus", s.DollarsAndTenthsOfCents(metricsEBSPrice)})
 	rows = append(rows, []interface{}{"2 network load balancers", s.DollarsMaxPrecision(nlbPrice*2) + " total"})
 
 	if clusterConfig.NATGateway == clusterconfig.SingleNATGateway {

cli/cmd/cluster_gcp.go

@@ -467,7 +467,7 @@ func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Cli
 					},
 					ServiceAccount: gcpClient.ClientEmail,
 				},
-				InitialNodeCount: 1,
+				InitialNodeCount: 2,
 			},
 		},
 		Locations: []string{*clusterConfig.Zone},

cli/cmd/lib_cluster_config_aws.go

@@ -239,7 +239,7 @@ func getConfigureClusterConfig(cachedClusterConfig clusterconfig.Config, awsCred
 		}
 		promptIfNotAdmin(awsClient, disallowPrompt)
 
-		err = setConfigFieldsFromCached(userClusterConfig, &cachedClusterConfig, awsClient)
+		err = setConfigFieldsFromCached(userClusterConfig, &cachedClusterConfig)
 		if err != nil {
 			return nil, err
 		}
@@ -265,12 +265,12 @@ func getConfigureClusterConfig(cachedClusterConfig clusterconfig.Config, awsCred
 		return nil, err
 	}
 
-	confirmConfigureClusterConfig(*userClusterConfig, awsCreds, awsClient, disallowPrompt)
+	confirmConfigureClusterConfig(*userClusterConfig, awsCreds, disallowPrompt)
 
 	return userClusterConfig, nil
 }
 
-func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedClusterConfig *clusterconfig.Config, awsClient *aws.Client) error {
+func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedClusterConfig *clusterconfig.Config) error {
 	if userClusterConfig.Bucket != "" && userClusterConfig.Bucket != cachedClusterConfig.Bucket {
 		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.BucketKey, cachedClusterConfig.Bucket)
 	}
@@ -388,11 +388,6 @@ func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedCl
 	}
 	userClusterConfig.ImageFluentBit = cachedClusterConfig.ImageFluentBit
 
-	if s.Obj(cachedClusterConfig.ImageStatsd) != s.Obj(userClusterConfig.ImageStatsd) {
-		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageStatsdKey, cachedClusterConfig.ImageStatsd)
-	}
-	userClusterConfig.ImageStatsd = cachedClusterConfig.ImageStatsd
-
 	if s.Obj(cachedClusterConfig.ImageIstioProxy) != s.Obj(userClusterConfig.ImageIstioProxy) {
 		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageIstioProxyKey, cachedClusterConfig.ImageIstioProxy)
 	}
@@ -403,6 +398,26 @@ func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedCl
 	}
 	userClusterConfig.ImageIstioPilot = cachedClusterConfig.ImageIstioPilot
 
+	if s.Obj(cachedClusterConfig.ImagePrometheus) != s.Obj(userClusterConfig.ImagePrometheus) {
+		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImagePrometheusKey, cachedClusterConfig.ImagePrometheus)
+	}
+
+	if s.Obj(cachedClusterConfig.ImagePrometheusConfigReloader) != s.Obj(userClusterConfig.ImagePrometheusConfigReloader) {
+		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImagePrometheusConfigReloaderKey, cachedClusterConfig.ImagePrometheusConfigReloader)
+	}
+
+	if s.Obj(cachedClusterConfig.ImagePrometheusOperator) != s.Obj(userClusterConfig.ImagePrometheusOperator) {
+		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImagePrometheusOperatorKey, cachedClusterConfig.ImagePrometheusOperator)
+	}
+
+	if s.Obj(cachedClusterConfig.ImagePrometheusStatsDExporter) != s.Obj(userClusterConfig.ImagePrometheusStatsDExporter) {
+		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImagePrometheusStatsDExporterKey, cachedClusterConfig.ImagePrometheusStatsDExporter)
+	}
+
+	if s.Obj(cachedClusterConfig.ImagePrometheusToCloudWatch) != s.Obj(userClusterConfig.ImagePrometheusToCloudWatch) {
+		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImagePrometheusToCloudwatchKey, cachedClusterConfig.ImagePrometheusToCloudWatch)
+	}
+
 	if userClusterConfig.Spot != nil && *userClusterConfig.Spot != *cachedClusterConfig.Spot {
 		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.SpotKey, *cachedClusterConfig.Spot)
 	}
@@ -450,6 +465,7 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 	eksPrice := aws.EKSPrices[*clusterConfig.Region]
 	operatorInstancePrice := aws.InstanceMetadatas[*clusterConfig.Region]["t3.medium"].Price
 	operatorEBSPrice := aws.EBSMetadatas[*clusterConfig.Region]["gp2"].PriceGB * 20 / 30 / 24
+	metricsEBSPrice := aws.EBSMetadatas[*clusterConfig.Region]["gp2"].PriceGB * 40 / 30 / 24
 	nlbPrice := aws.NLBMetadatas[*clusterConfig.Region].Price
 	natUnitPrice := aws.NATMetadatas[*clusterConfig.Region].Price
 	apiInstancePrice := aws.InstanceMetadatas[*clusterConfig.Region][*clusterConfig.InstanceType].Price
@@ -465,7 +481,7 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 		natTotalPrice = natUnitPrice * float64(len(clusterConfig.AvailabilityZones))
 	}
 
-	fixedPrice := eksPrice + operatorInstancePrice + operatorEBSPrice + 2*nlbPrice + natTotalPrice
+	fixedPrice := eksPrice + 2*operatorInstancePrice + operatorEBSPrice + metricsEBSPrice + 2*nlbPrice + natTotalPrice
 	totalMinPrice := fixedPrice + float64(*clusterConfig.MinInstances)*(apiInstancePrice+apiEBSPrice)
 	totalMaxPrice := fixedPrice + float64(*clusterConfig.MaxInstances)*(apiInstancePrice+apiEBSPrice)
 
@@ -476,7 +492,7 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 		{Title: "cost per hour"},
 	}
 
-	rows := [][]interface{}{}
+	var rows [][]interface{}
 	rows = append(rows, []interface{}{"1 eks cluster", s.DollarsMaxPrecision(eksPrice)})
 
 	instanceStr := "instances"
@@ -505,8 +521,9 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 
 	rows = append(rows, []interface{}{workerInstanceStr, workerPriceStr})
 	rows = append(rows, []interface{}{ebsInstanceStr, s.DollarsAndTenthsOfCents(apiEBSPrice) + " each"})
-	rows = append(rows, []interface{}{"1 t3.medium instance for the operator", s.DollarsMaxPrecision(operatorInstancePrice)})
+	rows = append(rows, []interface{}{"2 t3.medium instances for the cortex system", s.DollarsMaxPrecision(operatorInstancePrice * 2)})
 	rows = append(rows, []interface{}{"1 20gb ebs volume for the operator", s.DollarsAndTenthsOfCents(operatorEBSPrice)})
+	rows = append(rows, []interface{}{"1 40gb ebs volume for prometheus", s.DollarsAndTenthsOfCents(metricsEBSPrice)})
 	rows = append(rows, []interface{}{"2 network load balancers", s.DollarsMaxPrecision(nlbPrice) + " each"})
 
 	if clusterConfig.NATGateway == clusterconfig.SingleNATGateway {
@@ -565,16 +582,16 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 	}
 }
 
-func confirmConfigureClusterConfig(clusterConfig clusterconfig.Config, awsCreds AWSCredentials, awsClient *aws.Client, disallowPrompt bool) {
-	fmt.Println(clusterConfigConfirmationStr(clusterConfig, awsCreds, awsClient))
+func confirmConfigureClusterConfig(clusterConfig clusterconfig.Config, awsCreds AWSCredentials, disallowPrompt bool) {
+	fmt.Println(clusterConfigConfirmationStr(clusterConfig, awsCreds))
 
 	if !disallowPrompt {
 		exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://docs.cortex.dev/v/%s/ for more information", consts.CortexVersionMinor)
 		prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be updated according to the configuration above, are you sure you want to continue?", clusterConfig.ClusterName, *clusterConfig.Region), "", exitMessage)
 	}
 }
 
-func clusterConfigConfirmationStr(clusterConfig clusterconfig.Config, awsCreds AWSCredentials, awsClient *aws.Client) string {
+func clusterConfigConfirmationStr(clusterConfig clusterconfig.Config, awsCreds AWSCredentials) string {
 	defaultConfig, _ := clusterconfig.GetDefaults()
 
 	var items table.KeyValuePairs
@@ -694,15 +711,26 @@ func clusterConfigConfirmationStr(clusterConfig clusterconfig.Config, awsCreds A
 	if clusterConfig.ImageFluentBit != defaultConfig.ImageFluentBit {
 		items.Add(clusterconfig.ImageFluentBitUserKey, clusterConfig.ImageFluentBit)
 	}
-	if clusterConfig.ImageStatsd != defaultConfig.ImageStatsd {
-		items.Add(clusterconfig.ImageStatsdUserKey, clusterConfig.ImageStatsd)
-	}
 	if clusterConfig.ImageIstioProxy != defaultConfig.ImageIstioProxy {
 		items.Add(clusterconfig.ImageIstioProxyUserKey, clusterConfig.ImageIstioProxy)
 	}
 	if clusterConfig.ImageIstioPilot != defaultConfig.ImageIstioPilot {
 		items.Add(clusterconfig.ImageIstioPilotUserKey, clusterConfig.ImageIstioPilot)
 	}
-
+	if clusterConfig.ImagePrometheus != defaultConfig.ImagePrometheus {
+		items.Add(clusterconfig.ImagePrometheusUserKey, clusterConfig.ImagePrometheus)
+	}
+	if clusterConfig.ImagePrometheusConfigReloader != defaultConfig.ImagePrometheusConfigReloader {
+		items.Add(clusterconfig.ImagePrometheusConfigReloaderUserKey, clusterConfig.ImagePrometheusConfigReloader)
+	}
+	if clusterConfig.ImagePrometheusOperator != defaultConfig.ImagePrometheusOperator {
+		items.Add(clusterconfig.ImagePrometheusOperatorUserKey, clusterConfig.ImagePrometheusOperator)
+	}
+	if clusterConfig.ImagePrometheusStatsDExporter != defaultConfig.ImagePrometheusStatsDExporter {
+		items.Add(clusterconfig.ImagePrometheusStatsDExporterUserKey, clusterConfig.ImagePrometheusStatsDExporter)
+	}
+	if clusterConfig.ImagePrometheusToCloudWatch != defaultConfig.ImagePrometheusToCloudWatch {
+		items.Add(clusterconfig.ImagePrometheusToCloudwatchUserKey, clusterConfig.ImagePrometheusToCloudWatch)
+	}
 	return items.String()
 }

cli/cmd/lib_cluster_config_gcp.go

@@ -54,15 +54,6 @@ func existingCachedGCPClusterConfigPaths() []string {
 	return matches
 }
 
-func readCachedGCPClusterConfigFile(clusterConfig *clusterconfig.GCPConfig, filePath string) error {
-	errs := cr.ParseYAMLFile(clusterConfig, clusterconfig.GCPValidation, filePath)
-	if errors.HasError(errs) {
-		return errors.FirstError(errs...)
-	}
-
-	return nil
-}
-
 func readUserGCPClusterConfigFile(clusterConfig *clusterconfig.GCPConfig) error {
 	errs := cr.ParseYAMLFile(clusterConfig, clusterconfig.UserGCPValidation, _flagClusterGCPConfig)
 	if errors.HasError(errs) {

dev/operator_local.sh

@@ -94,6 +94,13 @@ fi
 
 export CORTEX_OPERATOR_IN_CLUSTER=false
 export CORTEX_CLUSTER_CONFIG_PATH=~/.cortex/cluster-dev.yaml
+export CORTEX_PROMETHEUS_URL="http://localhost:9090"
+
+portForwardCMD="kubectl port-forward -n default prometheus-prometheus-0 9090"
+kill $(pgrep -f "${portForwardCMD}") >/dev/null 2>&1 || true
+
+echo "Port-forwarding Prometheus to localhost:9090"
+eval "${portForwardCMD}" >/dev/null 2>&1 &
 
 mkdir -p $ROOT/bin
 

dev/versions.md

@@ -115,7 +115,7 @@ _note: docker client installation may be able to be improved, see https://github
 
 ### request-monitor
 
-1. `cd images/request-monitor/`
+1. `cd request-monitor/`
 1. `rm -rf go.mod go.sum && go mod init && go clean -modcache`
 1. `go mod tidy`
 1. Check that the diff in `go.mod` is reasonable

docs/clusters/aws/install.md

@@ -99,7 +99,11 @@ image_inferentia: quay.io/cortexlabs/inferentia:master
 image_neuron_rtd: quay.io/cortexlabs/neuron-rtd:master
 image_nvidia: quay.io/cortexlabs/nvidia:master
 image_fluent_bit: quay.io/cortexlabs/fluent-bit:master
-image_statsd: quay.io/cortexlabs/statsd:master
 image_istio_proxy: quay.io/cortexlabs/istio-proxy:master
 image_istio_pilot: quay.io/cortexlabs/istio-pilot:master
+image_prometheus: quay.io/cortexlabs/prometheus:master
+image_prometheus_config_reloader: quay.io/cortexlabs/prometheus-config-reloader:master
+image_prometheus_operator: quay.io/cortexlabs/prometheus-operator:master
+image_prometheus_statsd_exporter: quay.io/cortexlabs/prometheus-statsd-exporter:master
+image_prometheus_to_cloudwatch: quay.io/cortexlabs/prometheus-to-cloudwatch:master
 ```

docs/clusters/gcp/install.md

@@ -74,8 +74,12 @@ The docker images used by the Cortex cluster can also be overridden, although th
 image_operator: quay.io/cortexlabs/operator:master
 image_manager: quay.io/cortexlabs/manager:master
 image_downloader: quay.io/cortexlabs/downloader:master
-image_statsd: quay.io/cortexlabs/statsd:master
 image_istio_proxy: quay.io/cortexlabs/istio-proxy:master
 image_istio_pilot: quay.io/cortexlabs/istio-pilot:master
 image_pause: quay.io/cortexlabs/pause:master
+image_prometheus: quay.io/cortexlabs/prometheus:master
+image_prometheus_config_reloader: quay.io/cortexlabs/prometheus-config-reloader:master
+image_prometheus_operator: quay.io/cortexlabs/prometheus-operator:master
+image_prometheus_statsd_exporter: quay.io/cortexlabs/prometheus-statsd-exporter:master
+image_prometheus_stackdriver_sidecar: quay.io/cortexlabs/prometheus-stackdriver-sidecar:master
 ```

go.mod

@@ -34,6 +34,8 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v0.9.3
+	github.com/prometheus/common v0.4.0
 	github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect
 	github.com/shirou/gopsutil v3.20.11+incompatible
 	github.com/sirupsen/logrus v1.4.1 // indirect