Skip to content

Commit a0d82be

Browse files
vishalbolluvishalbollu
committed
Update #2 
1 parent f4503b0 commit a0d82be

File tree

7 files changed

+128
-171
lines changed

7 files changed

+128
-171
lines changed

‎examples/iris/color.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"samples": [
3+
{
4+
"sepal_length": 5.5,
5+
"sepal_width": 2.4,
6+
"petal_length": 3.7,
7+
"petal_width": 1
8+
}
9+
]
10+
}

‎examples/iris/setosa.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"samples": [
3+
{
4+
"sepal_length": 5,
5+
"sepal_width": 3.3,
6+
"petal_length": 1.4,
7+
"petal_width": 0.2
8+
}
9+
]
10+
}

‎examples/iris/virginica.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"samples": [
3+
{
4+
"sepal_length": 5.6,
5+
"sepal_width": 2.8,
6+
"petal_length": 4.9,
7+
"petal_width": 2
8+
}
9+
]
10+
}

‎manager/manifests/istio-metrics.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
params:
2323
value: response.duration
2424
dimensions:
25-
REQUEST_PATH: request.url_path | "unknown"
25+
RequestPath: request.url_path | "unknown"
2626
---
2727
apiVersion: config.istio.io/v1alpha2
2828
kind: handler

‎pkg/operator/endpoints/metrics.go

Lines changed: 22 additions & 169 deletions
Original file line numberDiff line numberDiff line change
@@ -39,19 +39,34 @@ func GetMetrics(w http.ResponseWriter, r *http.Request) {
3939
}
4040

4141
ctx := workloads.CurrentContext(appName)
42+
err := workloads.PopulateWorkloadIDs(ctx)
43+
if err != nil {
44+
RespondError(w, err)
45+
return
46+
}
47+
4248
apiName, err := getRequiredQueryParam("apiName", r)
4349
if err != nil {
4450
RespondError(w, err)
4551
return
4652
}
4753

54+
api := ctx.APIs[apiName]
55+
apiSavedStatus, err := workloads.GetAPISavedStatus(api.ID, api.WorkloadID, appName)
56+
if err != nil {
57+
RespondError(w, err)
58+
return
59+
}
60+
4861
endTime := time.Now()
62+
twoWeeksAgo := endTime.Add(- 14 * 24 * time.Hour)
63+
metricsStartTime := apiSavedStatus.Start
64+
if twoWeeksAgo
65+
4966
period := int64(60 * 60)
5067

51-
//GetClassificationMetrics(appName, apiName, ctx.APIs[apiName].ID, period, &endTime)
5268
if *ctx.APIs[apiName].Tracker.ModelType == "classification" {
5369
metrics := GetClassificationMetrics(appName, apiName, ctx.APIs[apiName].ID, period, &endTime)
54-
// GetNetworkStats(appName, apiName, ctx.APIs[apiName].ID, stats)
5570
Respond(w, metrics)
5671
} else {
5772
metrics := GetRegressionMetrics(appName, apiName, ctx.APIs[apiName].ID, period, &endTime)
@@ -104,15 +119,12 @@ func Max(floats ...*float64) *float64 {
104119
}
105120

106121
func Avg(values []*float64, counts []*float64) *float64 {
107-
debug.Pp(values)
108-
debug.Pp(counts)
109122
if len(values) == 0 || len(counts) == 0 {
110123
return nil
111124
}
112125

113126
total := float64(*SumInt(counts...))
114127
avg := 0.0
115-
debug.Pp(total)
116128
for idx, valPtr := range values {
117129
weight := *counts[idx]
118130
value := *valPtr
@@ -167,6 +179,10 @@ func GetClassificationMetrics(appName string, apiName string, apiID string, peri
167179
return &apiMetrics
168180
}
169181

182+
func GetLatencyMetricsDef(routeName string, period int64, endTime *time.Time) {
183+
184+
}
185+
170186
func GetRegressionMetrics(appName string, apiName string, apiID string, period int64, endTime *time.Time) *schema.APIMetrics {
171187
startTime := endTime.Add(-7 * 24 * 60 * 60 * time.Second)
172188

@@ -281,29 +297,6 @@ func GetRegressionMetricDef(appName string, apiName string, apiID string, period
281297

282298
return regressionMetric
283299
}
284-
// input := cloudwatch.GetMetricStatisticsInput{
285-
// EndTime: &endTime,
286-
// StartTime: &startTime,
287-
// Namespace: aws.String("cortex"),
288-
// MetricName: aws.String("PREDICTION"),
289-
// Dimensions: []*cloudwatch.Dimension{
290-
// &cloudwatch.Dimension{
291-
// Name: aws.String("AppName"),
292-
// Value: aws.String(appName),
293-
// },
294-
// &cloudwatch.Dimension{
295-
// Name: aws.String("APIName"),
296-
// Value: aws.String(apiName),
297-
// },
298-
// &cloudwatch.Dimension{
299-
// Name: aws.String("APIID"),
300-
// Value: aws.String(apiID),
301-
// },
302-
// },
303-
// Period: aws.Int64(5 * 60),
304-
// Statistics: []*string{aws.String("SampleCount"), aws.String("Minimum"), aws.String("Maximum"), aws.String("Average")},
305-
// }
306-
//}
307300

308301
func GetNetworkStatsDef(appName string, apiName string, apiID string, period int64) []*cloudwatch.MetricDataQuery {
309302
dimensions := []*cloudwatch.Dimension{
@@ -438,144 +431,4 @@ func GetClassesMetricDef(appName string, apiName string, apiID string, period in
438431
}
439432
debug.Pp(classMetricQueries)
440433
return classMetricQueries, nil
441-
}
442-
443-
// func GetNetworkStatsMetricDef(appName string, apiName string, apiID string) ([]*cloudwatch.Metric, error) {
444-
445-
// }
446-
447-
// func GetClassificationMetrics(appName string, apiName string, apiID string) (*schema.PredictionMetrics, error) {
448-
// var predictionMetrics schema.PredictionMetrics
449-
450-
// classMetricQuery, err := GetClassesMetricDef(appName, apiName, apiID)
451-
// if err != nil {
452-
// return err
453-
// }
454-
455-
// startTime := time.Now().Add(-5 * 60 * time.Second)
456-
// endTime := time.Now()
457-
// classMapping := make(map[string]float64, len(metricsList))
458-
459-
// for i := 0; i < metricDataQueryBatchCount; i++ {
460-
// var batchSize int
461-
// if i == metricDataQueryBatchCount - 1{
462-
// batchSize = numClasses % 100
463-
// } else {
464-
// batchSize = 100
465-
// }
466-
// queries := make([]*cloudwatch.MetricDataQuery, batchSize)
467-
468-
// prevPage := i * 100
469-
// for metricID := prevPage; metricID < prevPage + batchSize; metricID++ {
470-
// queries[metricID-prevPage] = &cloudwatch.MetricDataQuery{
471-
// Id: aws.String(fmt.Sprintf("id_%d", metricID)),
472-
// MetricStat: &cloudwatch.MetricStat{
473-
// Metric: metricsList[metricID],
474-
// Period: aws.Int64(5 * 60),
475-
// Stat: aws.String("Sum"),
476-
// },
477-
// }
478-
// }
479-
480-
// metricsDataQuery := cloudwatch.GetMetricDataInput{
481-
// EndTime: &endTime,
482-
// StartTime: &startTime,
483-
// MetricDataQueries: queries,
484-
// }
485-
486-
// output, _ := config.AWS.CloudWatchMetrics.GetMetricData(&metricsDataQuery)
487-
// debug.Pp(output)
488-
489-
// for _, result := range output.MetricDataResults {
490-
// idList := strings.Split(*result.Id, "_")
491-
// id, _ := strconv.Atoi(idList[1])
492-
// dims := metricsList[id].Dimensions
493-
// var metricName string
494-
// for _, dim := range dims {
495-
// if *dim.Name == "CLASS" {
496-
// metricName = *dim.Value
497-
// }
498-
// }
499-
// if len(result.Values) == 0 {
500-
// classMapping[metricName] = 0
501-
// } else {
502-
// classMapping[metricName] = *result.Values[0]
503-
// }
504-
// }
505-
// }
506-
507-
// startTimeSecond := startTime.Round(time.Minute)
508-
// if startTimeSecond.After(startTime) {
509-
// startTimeSecond.Add(-1 * time.Minute)
510-
// }
511-
// endTimeSecond := startTimeSecond.Add(5 * time.Minute)
512-
513-
// debug.Pp(startTimeSecond)
514-
515-
// predictionMetrics.StartTime = &startTimeSecond
516-
// predictionMetrics.EndTime = &endTimeSecond
517-
// predictionMetrics.ClassDistribution = classMapping
518-
// return &predictionMetrics
519-
// }
520-
521-
// func GetRegressionMetrics(appName string, apiName string, apiID string) *schema.PredictionMetrics {
522-
// fmt.Println("GetRegressionMetrics")
523-
// var predictionMetrics schema.PredictionMetrics
524-
// startTime := time.Now().Add(-5 * 60 * time.Second)
525-
// endTime := time.Now()
526-
// input := cloudwatch.GetMetricStatisticsInput{
527-
// EndTime: &endTime,
528-
// StartTime: &startTime,
529-
// Namespace: aws.String("cortex"),
530-
// MetricName: aws.String("PREDICTION"),
531-
// Dimensions: []*cloudwatch.Dimension{
532-
// &cloudwatch.Dimension{
533-
// Name: aws.String("AppName"),
534-
// Value: aws.String(appName),
535-
// },
536-
// &cloudwatch.Dimension{
537-
// Name: aws.String("APIName"),
538-
// Value: aws.String(apiName),
539-
// },
540-
// &cloudwatch.Dimension{
541-
// Name: aws.String("APIID"),
542-
// Value: aws.String(apiID),
543-
// },
544-
// },
545-
// Period: aws.Int64(5 * 60),
546-
// Statistics: []*string{aws.String("SampleCount"), aws.String("Minimum"), aws.String("Maximum"), aws.String("Average")},
547-
// }
548-
549-
// debug.Pp(input)
550-
// response, _ := config.AWS.CloudWatchMetrics.GetMetricStatistics(&input)
551-
// debug.Pp(response)
552-
// if response.Datapoints == nil || len(response.Datapoints) == 0 {
553-
// return &predictionMetrics
554-
// }
555-
556-
// predictionMetrics.StartTime = response.Datapoints[0].Timestamp
557-
// dataPointStartTime := *response.Datapoints[0].Timestamp
558-
// dataPointEndTime := dataPointStartTime.Add(5 * 60 * time.Second)
559-
560-
// predictionMetrics.StartTime = &dataPointStartTime
561-
// predictionMetrics.EndTime = &dataPointEndTime
562-
563-
// basicStats := map[string]float64{}
564-
// basicStats["SampleCount"] = *response.Datapoints[0].SampleCount
565-
// basicStats["Minimum"] = *response.Datapoints[0].Minimum
566-
// basicStats["Maximum"] = *response.Datapoints[0].Maximum
567-
// basicStats["Average"] = *response.Datapoints[0].Average
568-
// predictionMetrics.Statistics = basicStats
569-
570-
// if *response.Datapoints[0].Minimum > 0 {
571-
// percentiles := map[string]float64{}
572-
// input.Statistics = nil
573-
// input.ExtendedStatistics = []*string{aws.String("p25.0"), aws.String("p50.0"), aws.String("p75.0")}
574-
// response, _ := config.AWS.CloudWatchMetrics.GetMetricStatistics(&input)
575-
// for name, val := range response.Datapoints[0].ExtendedStatistics{
576-
// percentiles[name] = *val
577-
// }
578-
// predictionMetrics.Percentiles = percentiles
579-
// }
580-
// return &predictionMetrics
581-
// }
434+
}

‎pkg/operator/workloads/api_saved_status.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func uploadAPISavedStatusFunc(savedStatus *resource.APISavedStatus) func() error
5959
}
6060
}
6161

62-
func getAPISavedStatus(resourceID string, workloadID string, appName string) (*resource.APISavedStatus, error) {
62+
func GetAPISavedStatus(resourceID string, workloadID string, appName string) (*resource.APISavedStatus, error) {
6363
if cachedSavedStatus, ok := getCachedAPISavedStatus(resourceID, workloadID, appName); ok {
6464
return cachedSavedStatus, nil
6565
}

‎pkg/workloads/cortex/lib/api_utils.py

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# Copyright 2019 Cortex Labs, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
from cortex.lib.exceptions import UserException
17+
18+
19+
def status_code_metric(dimensions, status_code):
20+
status_code_series = int(status_code / 100)
21+
dimensions.append({"Name": "Code", "Value": "{}XX".format(status_code_series)})
22+
return [{"MetricName": "StatusCode", "Dimensions": dimensions, "Value": 1}]
23+
24+
25+
def predictions_per_request_metric(dimensions, prediction_count):
26+
return [
27+
{"MetricName": "PredictionsPerRequest", "Dimensions": dimensions, "Value": prediction_count}
28+
]
29+
30+
31+
def prediction_metrics(dimensions, api, predictions):
32+
metric_list = []
33+
tracker = api.get("tracker")
34+
for prediction in predictions:
35+
predicted_value = prediction["prediction"].get(tracker["key"])
36+
if predicted_value is None:
37+
raise UserException(
38+
"key {} not found in prediction".format(tracker["key"]), api["name"]
39+
)
40+
41+
if tracker["model_type"] == "classification":
42+
if type(predicted_value) != str:
43+
raise UserException(
44+
"expected type 'str' but found '{}' when tracking key '{}'".format(
45+
type(predicted_value), tracker["key"]
46+
),
47+
api["name"],
48+
)
49+
dimensions_with_class = dimensions.copy()
50+
dimensions_with_class.append({"Name": "Class", "Value": str(predicted_value)})
51+
metric = {
52+
"MetricName": "Prediction",
53+
"Dimensions": dimensions,
54+
"Unit": "Count",
55+
"Value": 1,
56+
}
57+
58+
metric_list.append(metric)
59+
60+
else:
61+
if type(predicted_value) != float:
62+
raise UserException(
63+
"expected type 'float' but found '{}' when tracking key '{}'".format(
64+
type(predicted_value), tracker["key"]
65+
),
66+
api["name"],
67+
)
68+
metric = {
69+
"MetricName": "Prediction",
70+
"Dimensions": dimensions,
71+
"Value": predicted_value,
72+
}
73+
metric_list.append(metric)
74+
return metric_list

0 commit comments

Comments
 (0)