Skip to content
This repository has been archived by the owner on Mar 20, 2023. It is now read-only.

Commit

Permalink
Merge pull request #98 from microsoft/custom-metrics-hpa
Browse files Browse the repository at this point in the history
Custom metrics hpa
  • Loading branch information
cmcheca authored Nov 26, 2020
2 parents b087d1d + 30bae71 commit 9d889e9
Show file tree
Hide file tree
Showing 36 changed files with 301 additions and 49 deletions.
49 changes: 36 additions & 13 deletions Deploy/Deploy-Unified.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,13 @@ Param(
[parameter(Mandatory=$true)][string]$subscription,
[parameter(Mandatory=$false)][string]$clientId,
[parameter(Mandatory=$false)][string]$password,
[parameter(Mandatory=$false)][bool]$deployAks=$true
[parameter(Mandatory=$false)][bool]$deployAks=$true,
[parameter(Mandatory=$false)][bool]$deployProductsOnVnodes=$false,
[parameter(Mandatory=$false)][bool]$stepDeployArm=$true,
[parameter(Mandatory=$false)][bool]$stepBuildPush=$true,
[parameter(Mandatory=$false)][bool]$stepDeployImages=$true,
[parameter(Mandatory=$false)][bool]$stepDeployPics=$true,
[parameter(Mandatory=$false)][bool]$stepLoginAzure=$true
)
$gValuesFile="configFile.yaml"

Expand All @@ -16,16 +22,23 @@ Push-Location $($MyInvocation.InvocationName | Split-Path)
az extension add --name aks-preview
az extension update --name aks-preview

# Write-Host "Login in your account" -ForegroundColor Yellow
az login
az extension add --name application-insights
az extension update --name application-insights

if ($stepLoginAzure) {
# Write-Host "Login in your account" -ForegroundColor Yellow
az login
}

# Write-Host "Choosing your subscription" -ForegroundColor Yellow
az account set --subscription $subscription

Push-Location powershell

# Deploy ARM
& ./Deploy-Arm-Azure.ps1 -resourceGroup $resourceGroup -location $location -clientId $clientId -password $password -deployAks $deployAks
if ($stepDeployArm) {
# Deploy ARM
& ./Deploy-Arm-Azure.ps1 -resourceGroup $resourceGroup -location $location -clientId $clientId -password $password -deployAks $deployAks
}

# Connecting kubectl to AKS
Write-Host "Retrieving Aks Name" -ForegroundColor Yellow
Expand All @@ -44,16 +57,26 @@ $acrName = $(az acr list --resource-group $resourceGroup --subscription $subscri
Write-Host "The Name of your ACR: $acrName" -ForegroundColor Yellow
& ./Create-Secret.ps1 -resourceGroup $resourceGroup -acrName $acrName

# Build an Push
& ./Build-Push.ps1 -resourceGroup $resourceGroup -acrName $acrName -isWindows $false
if ($stepBuildPush) {
# Build an Push
& ./Build-Push.ps1 -resourceGroup $resourceGroup -acrName $acrName -isWindows $false
}

# Deploy images in AKS
$gValuesLocation=$(./Join-Path-Recursively.ps1 -pathParts __values,$gValuesFile)
& ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts "*" -acrName $acrName -valuesFile $gValuesLocation
if ($stepDeployImages) {
# Deploy images in AKS
$gValuesLocation=$(./Join-Path-Recursively.ps1 -pathParts __values,$gValuesFile)
$chartsToDeploy = "*"
if ($deployProductsOnVnodes) {
$chartsToDeploy = "*,pr!" #pr! forces using vnodes for products chart
}
& ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts $chartsToDeploy -acrName $acrName -valuesFile $gValuesLocation
}

# Deploy pictures in AKS
$storageName = $(az resource list --resource-group $resourceGroup --resource-type Microsoft.Storage/storageAccounts -o json | ConvertFrom-Json).name
& ./Deploy-Pictures-Azure.ps1 -resourceGroup $resourceGroup -storageName $storageName
if ($stepDeployPics) {
# Deploy pictures in AKS
$storageName = $(az resource list --resource-group $resourceGroup --resource-type Microsoft.Storage/storageAccounts -o json | ConvertFrom-Json).name
& ./Deploy-Pictures-Azure.ps1 -resourceGroup $resourceGroup -storageName $storageName
}

Pop-Location
Pop-Location
19 changes: 11 additions & 8 deletions Deploy/demos/vnodes/BlackFriday.ps1
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
Param(
[parameter(Mandatory=$true)][string]$aksName,
[parameter(Mandatory=$false)][string]$aksHost,
[parameter(Mandatory=$false)][string]$aksName,
[parameter(Mandatory=$true)][string]$resourceGroup
)

Expand Down Expand Up @@ -32,14 +33,16 @@ workflow LoadTestUri {
}


$aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpapplicationrouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
if (-not $aksHost) {
$aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
}
if ([String]::IsNullOrEmpty($aksHost)) {
$aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpapplicationrouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
if (-not $aksHost) {
$aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
}

if (-not $aksHost) {
Write-Host "Could not infer URL for AKS $aksName in RG $resourceGroup"
exit 1
if (-not $aksHost) {
Write-Host "Could not infer URL for AKS $aksName in RG $resourceGroup"
exit 1
}
}

$url="http://$aksHost/product-api/v1/product"
Expand Down
32 changes: 32 additions & 0 deletions Deploy/demos/vnodes/blackfriday.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/bin/bash

dns=""

while [ "$1" != "" ]; do
case $1 in
-d | --dns) shift
dns=$1
;;
* ) echo "Invalid param. Use -d (ingress DNS)"
exit 1
esac
shift
done

if [ -z "$dns" ]; then
echo "No public ingress DNS provided. Use -d to provide it (no http prefix)"
exit 1
fi


while :
do
id=$(shuf -i 1-100 -n 1)
if (($id > 50)); then
echo "# Calling http:$dns/product-api/v1/product"
curl -s "http://$dns/product-api/v1/product" > /dev/null
else
echo "! Calling http:$dns/product-api/v1/product/$id"
curl -s "http://$dns/product-api/v1/product/$id" > /dev/null
fi
done
16 changes: 16 additions & 0 deletions Deploy/demos/vnodes/hpa/hpa-products.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
name: products-reqs-progress-hpa
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: tailwindtraders-product
minReplicas: 1
maxReplicas: 20
metrics:
- type: Pods
pods:
metricName: http_requests_in_progress
targetAverageValue: 3
Empty file added Deploy/demos/vnodes/null
Empty file.
18 changes: 18 additions & 0 deletions Deploy/demos/vnodes/prometheus-adapter/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Url to access prometheus
prometheus:
url: http://prometheus-operated.default.svc.cluster.local
port: 9090

metricsRelistInterval: 30s
logLevel: 6
rules:
default: false
custom:
- seriesQuery: 'http_requests_in_progress{namespace!="",pod!=""}'
resources:
overrides:
namespace: {resource: "namespace"}
pod: {resource: "pod"}
name:
as: "http_requests_in_progress"
metricsQuery: 'sum(avg_over_time(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)'
79 changes: 77 additions & 2 deletions Deploy/demos/vnodes/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,81 @@

This folder contains additional scripts needed for testing the vnodes scenario

## Powershell scripts

- `Create-Aks.ps1`: Creates an AKS with vnodes enabled.
## Pre steps

1. Install prometheus operator: `kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/master/bundle.yaml`
2. Run prometheus instance: `kubectl apply -f yamls`
3. Deploy Prometheus Metrics Adapter:
* `helm repo add prometheus-community https://prometheus-community.github.io/helm-charts`
* `helm repo update`
* `helm install prometheus-adapter prometheus-community/prometheus-adapter -f ./prometheus-adapter/values.yaml`
4. Deploy the service monitor to monitor products api: `kubectl apply -f service-monitor`
5. Create the HPA for products: `kubectl apply -f hpa`

## Verify metrics server

Verify metrics are exposed through metrics server: `kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/ | jq`

```json
{
"kind": "APIResourceList",
"apiVersion": "v1",
"groupVersion": "custom.metrics.k8s.io/v1beta1",
"resources": [
{
"name": "namespaces/http_requests_in_progress",
"singularName": "",
"namespaced": false,
"kind": "MetricValueList",
"verbs": [
"get"
]
},
{
"name": "pods/http_requests_in_progress",
"singularName": "",
"namespaced": true,
"kind": "MetricValueList",
"verbs": [
"get"
]
}
]
}
```

Verify metrics can be fetched from Prometheus: `kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespace/default/http_requests_in_progress`

```json
{
"kind": "MetricValueList",
"apiVersion": "custom.metrics.k8s.io/v1beta1",
"metadata": {
"selfLink": "/apis/custom.metrics.k8s.io/v1beta1/namespace/default/http_requests_in_progress"
},
"items": [
{
"describedObject": {
"kind": "Namespace",
"name": "default",
"apiVersion": "/v1"
},
"metricName": "http_requests_in_progress",
"timestamp": "2020-10-09T09:02:36Z",
"value": "1",
"selector": null
}
]
}
```

## Run blackfriday scripts

Now we need to do some calls to the products API. You can use the blackfriday script. Run several of them (`./blackfriday.sh -d <dns-ingress> /dev/null &`) and see how metric is growing in the HPA (`kubectl get hpa kubectl get hpa products-reqs-progress-hpa -w`).

Once demo is finished kill the blackfriday scripts:

`kill $(ps | grep blackfriday.sh | awk -F" " '{print $1}' )`

After some time HPA will scale down the replicas.
12 changes: 12 additions & 0 deletions Deploy/demos/vnodes/service-monitor/sm-products.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: sm-products
labels:
app: tailwindtraders
spec:
selector:
matchLabels:
app: tailwindtraders-products
endpoints:
- port: http
4 changes: 4 additions & 0 deletions Deploy/demos/vnodes/yamls/1-service-account.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
18 changes: 18 additions & 0 deletions Deploy/demos/vnodes/yamls/2-cluster-role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups: [""]
resources:
- nodes
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
12 changes: 12 additions & 0 deletions Deploy/demos/vnodes/yamls/3-cluster-role-binding.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: default
13 changes: 13 additions & 0 deletions Deploy/demos/vnodes/yamls/4-prometheus.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: prometheus
labels:
prometheus: prometheus
spec:
replicas: 1
scrapeInterval: 5s
serviceAccountName: prometheus
serviceMonitorSelector:
matchLabels:
app: tailwindtraders
2 changes: 1 addition & 1 deletion Deploy/helm/cart-api/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,6 @@ data:
policyName: {{ .Values.B2C.policyName | quote }}
identityMetadata: {{ .Values.B2C.identityMetadata | quote }}
issuer: {{ .Values.B2C.issuer | quote }}
APPLICATIONINSIGHTSIK: {{ .Values.inf.appinsights.id }}
APPLICATIONINSIGHTSIK: "{{ .Values.inf.appinsights.id }}"


2 changes: 1 addition & 1 deletion Deploy/helm/coupons-api/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ data:
URL_BASE: {{ $imagebaseurl }}
ISSUER: {{ .Values.sec.issuer }}
SECURITYKEY: {{ .Values.sec.securityKey }}
APPLICATIONINSIGHTSIK: {{ .Values.inf.appinsights.id }}
APPLICATIONINSIGHTSIK: "{{ .Values.inf.appinsights.id }}"
2 changes: 1 addition & 1 deletion Deploy/helm/image-classifier-api/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
data:
ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }}
ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}"
ImagePath: /imguploads
ISSUER: {{ .Values.sec.issuer }}
SECURITYKEY: {{ .Values.sec.securityKey }}
Expand Down
2 changes: 1 addition & 1 deletion Deploy/helm/login-api/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ data:
ProfilesImageUrl: {{ .Values.inf.storage.profileimages }}
Issuer: {{ .Values.sec.issuer }}
SecurityKey: {{ .Values.sec.securityKey }}
ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }}
ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}"
2 changes: 1 addition & 1 deletion Deploy/helm/mobilebff/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
data:
ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }}
ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}"
PopularProductsApiUrl: {{ .Values.inf.apiurls.popularproductsapiurl }}
ProductsApiUrl: {{ .Values.inf.apiurls.productsapiurl }}
ProfileApiUrl: {{ .Values.inf.apiurls.profileapiurl }}
Expand Down
2 changes: 1 addition & 1 deletion Deploy/helm/popular-products-api/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ metadata:
data:
AUTHKEY: {{ .Values.inf.db.popularproducts.auth }}
HOST: {{ .Values.inf.db.popularproducts.host }}
ApplicationInsightsIK: {{ .Values.inf.appinsights.id }}
ApplicationInsightsIK: "{{ .Values.inf.appinsights.id }}"
AzureStorageUrl: {{ .Values.inf.storage.productimages }}
ISSUER: {{ .Values.sec.issuer }}
SECURITYKEY: {{ .Values.sec.securityKey }}
2 changes: 1 addition & 1 deletion Deploy/helm/products-api/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ data:
CosmosDb__Host: {{ .Values.inf.db.products.host }}
ProductImagesUrl: {{ .Values.inf.storage.productimages }}
ProductDetailImagesUrl: {{ .Values.inf.storage.productdetailimages }}
ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }}
ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}"
ISSUER: {{ .Values.sec.issuer }}
SECURITYKEY: {{ .Values.sec.securityKey }}
Loading

0 comments on commit 9d889e9

Please sign in to comment.