Add metrics from Prometheus

modules/python/clusterloader2/autoscale/autoscale.py

@@ -94,7 +94,18 @@ def override_config_clusterloader2(cpu_per_node, node_count, pod_count, scale_up
     file.close()
 
 def execute_clusterloader2(cl2_image, cl2_config_dir, cl2_report_dir, kubeconfig, provider, cl2_config_file="config.yaml"):
-    run_cl2_command(kubeconfig, cl2_image, cl2_config_dir, cl2_report_dir, provider, cl2_config_file, overrides=True)
+    run_cl2_command(
+        kubeconfig,
+        cl2_image,
+        cl2_config_dir,
+        cl2_report_dir,
+        provider,
+        cl2_config_file,
+        overrides=True,
+        enable_prometheus=True,
+        scrape_ksm=True,
+        tear_down_prometheus=False
+    )
 
 def collect_clusterloader2(
     cpu_per_node,

modules/python/clusterloader2/autoscale/config/scale_pod_config.yaml

@@ -76,6 +76,33 @@ steps:
       labelSelector: {{$podLabelSelector}}
       timeout: {{$scaleUpTimeout}}
       refreshInterval: {{$refreshInterval}}
+- name: Capture Metrics After Scale Up {{$i}}
+  measurements:
+  - Identifier: ResourceMetrics{{$i}}
+    Method: GenericPrometheusQuery
+    Params:
+      action: start
+      metricName: Resource Metrics Summary
+      metricVersion: v1
+      unit: mixed
+      queries:
+      # Node Level Summary
+      - name: TotalNodes
+        query: count(kube_node_status_allocatable{resource="cpu"})
+      - name: NodeCPUAllocatable
+        query: sum(kube_node_status_allocatable{resource="cpu"})
+      - name: NodeMemoryAllocatable
+        query: sum(kube_node_status_allocatable{resource="memory"})
+      # Pod Level Summary
+      - name: TotalPods
+        query: count(kube_pod_status_phase{phase="Running"})
+      # Pod Distribution Summary
+      - name: PodsPerNodeAvg
+        query: avg(count by (node) (kube_pod_info{node!=""}))
+      - name: PodsPerNodeMax
+        query: max(count by (node) (kube_pod_info{node!=""}))
+      - name: PodsPerNodeMin
+        query: min(count by (node) (kube_pod_info{node!=""}))
 - name: Gather Measurements {{$i}}
   measurements:
   - Identifier: PodStartupLatency
@@ -90,6 +117,10 @@ steps:
     Method: ResourceUsageSummary
     Params:
       action: gather
+  - Identifier: ResourceMetrics{{$i}}
+    Method: GenericPrometheusQuery
+    Params:
+      action: gather
 - name: WaitBeforeDelete
   measurements:
   - Identifier: WaitBeforeDelete

modules/python/clusterloader2/utils.py

@@ -32,6 +32,10 @@ def run_cl2_command(kubeconfig, cl2_image, cl2_config_dir, cl2_report_dir, provi
 --enable-exec-service={enable_exec_service}
 --enable-prometheus-server={enable_prometheus}
 --prometheus-scrape-kubelets={scrape_kubelets}
+--prometheus-pvc-storage-class="managed"
+--prometheus-storage-class-provisioner="kubernetes.io/azure-disk"
+--prometheus-storage-class-volume-type="StandardSSD_LRS"
+--prometheus-memory-request="2Gi"
 --kubeconfig /root/.kube/config
 --testconfig /root/perf-tests/clusterloader2/config/{cl2_config_file}
 --report-dir /root/perf-tests/clusterloader2/results