Merge pull request kubernetes#13187 from spowelljr/addVirtualBoxBench…

…mark Add VirtualBox benchmarks to time-to-k8s daily benchmark
presztak · Dec 17, 2021 · f921731 · f921731
2 parents ef06107 + 3b5be82
commit f921731
Show file tree

Hide file tree

Showing 10 changed files with 76 additions and 28 deletions.
diff --git a/.github/workflows/time-to-k8s-public-chart.yml b/.github/workflows/time-to-k8s-public-chart.yml
@@ -8,7 +8,7 @@ env:
   GOPROXY: https://proxy.golang.org
   GO_VERSION: '1.17.5'
 jobs:
-  time-to-k8s-public-chart:
+  time-to-k8s-public-chart-docker:
     if: github.repository == 'kubernetes/minikube'
     runs-on: ubuntu-20.04
     env:
@@ -21,9 +21,32 @@ jobs:
         with:
           go-version: ${{env.GO_VERSION}}
           stable: true
-      - name: Benchmark time-to-k8s for Docker
+      - name: Benchmark time-to-k8s for Docker driver with Docker runtime
         run: |
-          ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh docker
-      - name: Benchmark time-to-k8s for Containerd
+          ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh docker docker
+      - name: Benchmark time-to-k8s for Docker driver with containerd runtime
         run: |
-          ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh containerd
+          ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh docker containerd
+  time-to-k8s-public-chart-virtualbox:
+    if: github.repository == 'kubernetes/minikube'
+    run-on: macos-10.15
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: 'us-west-1'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{env.GO_VERSION}}
+          stable: true
+      - name: Disable firewall
+        run: |
+          sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
+          sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k
+      - name: Benchmark time-to-k8s for VirtualBox driver with Docker runtime
+        run: |
+          ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh virtualbox docker
+      - name: Benchmark time-to-k8s for VirtualBox driver with containerd runtime
+        run: |
+          ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh virtualbox containerd
diff --git a/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml
diff --git a/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml
diff --git a/hack/benchmark/time-to-k8s/public-chart/docker-containerd-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/docker-containerd-benchmark.yaml
@@ -0,0 +1,4 @@
+testcases:
+  minikube:
+    setup: minikube start --driver=docker --container-runtime=containerd --memory=max --cpus=max
+    teardown: minikube delete
diff --git a/hack/benchmark/time-to-k8s/public-chart/docker-docker-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/docker-docker-benchmark.yaml
@@ -0,0 +1,4 @@
+testcases:
+  minikube:
+    setup: minikube start --driver=docker --container-runtime=docker --memory=max --cpus=max
+    teardown: minikube delete
diff --git a/hack/benchmark/time-to-k8s/public-chart/public-chart.sh b/hack/benchmark/time-to-k8s/public-chart/public-chart.sh
@@ -16,8 +16,9 @@
 
 set -e
 
+DRIVER="$1"
 # container-runtime (docker or containerd)
-RUNTIME="$1"
+RUNTIME="$2"
 BUCKET="s3://time-to-k8s"
 
 install_minikube() {
@@ -28,7 +29,7 @@ install_minikube() {
 run_benchmark() {
         ( cd ./hack/benchmark/time-to-k8s/time-to-k8s-repo/ &&
                 git submodule update --init &&
-                go run . --config "../public-chart/$RUNTIME-benchmark.yaml" --iterations 10 --output ./output.csv )
+                go run . --config "../public-chart/$DRIVER-$RUNTIME-benchmark.yaml" --iterations 10 --output ./output.csv )
 }
 
 generate_chart() {
@@ -46,16 +47,16 @@ cleanup() {
 	rm ./weekly-chart.png
 }
 
-copy "$BUCKET/$RUNTIME-runs.json" ./runs.json
+copy "$BUCKET/$DRIVER-$RUNTIME-runs.json" ./runs.json
 
 install_minikube
 
 run_benchmark
 generate_chart
 
-copy ./runs.json "$BUCKET/$RUNTIME-runs.json"
-copy ./runs.json "$BUCKET/$(date +'%Y-%m-%d')-$RUNTIME.json"
-copy ./daily-chart.png "$BUCKET/$RUNTIME-chart.png"
-copy ./weekly-chart.png "$BUCKET/$RUNTIME-weekly-chart.png"
+copy ./runs.json "$BUCKET/$DRIVER-$RUNTIME-runs.json"
+copy ./runs.json "$BUCKET/$(date +'%Y-%m-%d')-$DRIVER-$RUNTIME.json"
+copy ./daily-chart.png "$BUCKET/$DRIVER-$RUNTIME-chart.png"
+copy ./weekly-chart.png "$BUCKET/$DRIVER-$RUNTIME-weekly-chart.png"
 
 cleanup
diff --git a/hack/benchmark/time-to-k8s/public-chart/virtualbox-containerd-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/virtualbox-containerd-benchmark.yaml
@@ -0,0 +1,4 @@
+testcases:
+  minikube:
+    setup: minikube start --driver=virtualbox --container-runtime=containerd --memory=max --cpus=max
+    teardown: minikube delete
diff --git a/hack/benchmark/time-to-k8s/public-chart/virtualbox-docker-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/virtualbox-docker-benchmark.yaml
@@ -0,0 +1,4 @@
+testcases:
+  minikube:
+    setup: minikube start --driver=virtualbox --container-runtime=docker --memory=max --cpus=max
+    teardown: minikube delete
diff --git a/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md b/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md
@@ -7,10 +7,18 @@ weight: -99999999
 
 [Benchmarking Machine Specs](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources)
 
-## Docker
+## Docker driver - Docker runtime
 
-![Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-chart.png)
+![Docker Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-docker-chart.png)
 
-## Containerd
+## Docker driver - containerd runtime
 
-![Containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/containerd-chart.png)
+![Docker containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-containerd-chart.png)
+
+## VirtualBox driver - Docker runtime
+
+![VirtualBox Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-docker-chart.png)
+
+## VirtualBox driver - containerd runtime
+
+![VirtualBox containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-containerd-chart.png)
diff --git a/site/content/en/docs/benchmarks/timeToK8s/weekly_benchmark.md b/site/content/en/docs/benchmarks/timeToK8s/weekly_benchmark.md
@@ -7,10 +7,18 @@ weight: -99999998
 
 [Benchmarking Machine Specs](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources)
 
-## Docker
+## Docker driver - Docker runtime
 
-![Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-weekly-chart.png)
+![Docker Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-docker-weekly-chart.png)
 
-## Containerd
+## Docker driver - containerd runtime
 
-![Containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/containerd-weekly-chart.png)
+![Docker containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-containerd-weekly-chart.png)
+
+## VirtualBox driver - Docker runtime
+
+![VirtualBox Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-docker-weekly-chart.png)
+
+## VirtualBox driver - containerd runtime
+
+![VirtualBox containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-containerd-weekly-chart.png)