From 3b5be829c168815010d72bfcdcc0ae7fa7c94f21 Mon Sep 17 00:00:00 2001 From: Steven Powell Date: Thu, 16 Dec 2021 16:14:16 -0800 Subject: [PATCH] add VirtualBox benchmarks to time-to-k8s daily benchmark --- .../workflows/time-to-k8s-public-chart.yml | 33 ++++++++++++++++--- .../public-chart/containerd-benchmark.yaml | 4 --- .../public-chart/docker-benchmark.yaml | 4 --- .../docker-containerd-benchmark.yaml | 4 +++ .../public-chart/docker-docker-benchmark.yaml | 4 +++ .../time-to-k8s/public-chart/public-chart.sh | 15 +++++---- .../virtualbox-containerd-benchmark.yaml | 4 +++ .../virtualbox-docker-benchmark.yaml | 4 +++ .../benchmarks/timeToK8s/daily_benchmark.md | 16 ++++++--- .../benchmarks/timeToK8s/weekly_benchmark.md | 16 ++++++--- 10 files changed, 76 insertions(+), 28 deletions(-) delete mode 100644 hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml delete mode 100644 hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml create mode 100644 hack/benchmark/time-to-k8s/public-chart/docker-containerd-benchmark.yaml create mode 100644 hack/benchmark/time-to-k8s/public-chart/docker-docker-benchmark.yaml create mode 100644 hack/benchmark/time-to-k8s/public-chart/virtualbox-containerd-benchmark.yaml create mode 100644 hack/benchmark/time-to-k8s/public-chart/virtualbox-docker-benchmark.yaml diff --git a/.github/workflows/time-to-k8s-public-chart.yml b/.github/workflows/time-to-k8s-public-chart.yml index 7defbaf07e99..27999e6c3b6a 100644 --- a/.github/workflows/time-to-k8s-public-chart.yml +++ b/.github/workflows/time-to-k8s-public-chart.yml @@ -8,7 +8,7 @@ env: GOPROXY: https://proxy.golang.org GO_VERSION: '1.17.5' jobs: - time-to-k8s-public-chart: + time-to-k8s-public-chart-docker: if: github.repository == 'kubernetes/minikube' runs-on: ubuntu-20.04 env: @@ -21,9 +21,32 @@ jobs: with: go-version: ${{env.GO_VERSION}} stable: true - - name: Benchmark time-to-k8s for Docker + - name: Benchmark time-to-k8s for Docker driver with Docker runtime run: | - ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh docker - - name: Benchmark time-to-k8s for Containerd + ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh docker docker + - name: Benchmark time-to-k8s for Docker driver with containerd runtime run: | - ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh containerd + ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh docker containerd + time-to-k8s-public-chart-virtualbox: + if: github.repository == 'kubernetes/minikube' + run-on: macos-10.15 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: 'us-west-1' + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{env.GO_VERSION}} + stable: true + - name: Disable firewall + run: | + sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off + sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k + - name: Benchmark time-to-k8s for VirtualBox driver with Docker runtime + run: | + ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh virtualbox docker + - name: Benchmark time-to-k8s for VirtualBox driver with containerd runtime + run: | + ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh virtualbox containerd diff --git a/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml deleted file mode 100644 index 2fc4ff15ca83..000000000000 --- a/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml +++ /dev/null @@ -1,4 +0,0 @@ -testcases: - minikube: - setup: minikube start --container-runtime=containerd --memory=max --cpus=max - teardown: minikube delete diff --git a/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml deleted file mode 100644 index cfc224e1c268..000000000000 --- a/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml +++ /dev/null @@ -1,4 +0,0 @@ -testcases: - minikube: - setup: minikube start --container-runtime=docker --memory=max --cpus=max - teardown: minikube delete diff --git a/hack/benchmark/time-to-k8s/public-chart/docker-containerd-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/docker-containerd-benchmark.yaml new file mode 100644 index 000000000000..75aa52ecf839 --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/docker-containerd-benchmark.yaml @@ -0,0 +1,4 @@ +testcases: + minikube: + setup: minikube start --driver=docker --container-runtime=containerd --memory=max --cpus=max + teardown: minikube delete diff --git a/hack/benchmark/time-to-k8s/public-chart/docker-docker-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/docker-docker-benchmark.yaml new file mode 100644 index 000000000000..940c1c8f6537 --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/docker-docker-benchmark.yaml @@ -0,0 +1,4 @@ +testcases: + minikube: + setup: minikube start --driver=docker --container-runtime=docker --memory=max --cpus=max + teardown: minikube delete diff --git a/hack/benchmark/time-to-k8s/public-chart/public-chart.sh b/hack/benchmark/time-to-k8s/public-chart/public-chart.sh index 9448cc398954..e4194d680674 100755 --- a/hack/benchmark/time-to-k8s/public-chart/public-chart.sh +++ b/hack/benchmark/time-to-k8s/public-chart/public-chart.sh @@ -16,8 +16,9 @@ set -e +DRIVER="$1" # container-runtime (docker or containerd) -RUNTIME="$1" +RUNTIME="$2" BUCKET="s3://time-to-k8s" install_minikube() { @@ -28,7 +29,7 @@ install_minikube() { run_benchmark() { ( cd ./hack/benchmark/time-to-k8s/time-to-k8s-repo/ && git submodule update --init && - go run . --config "../public-chart/$RUNTIME-benchmark.yaml" --iterations 10 --output ./output.csv ) + go run . --config "../public-chart/$DRIVER-$RUNTIME-benchmark.yaml" --iterations 10 --output ./output.csv ) } generate_chart() { @@ -46,16 +47,16 @@ cleanup() { rm ./weekly-chart.png } -copy "$BUCKET/$RUNTIME-runs.json" ./runs.json +copy "$BUCKET/$DRIVER-$RUNTIME-runs.json" ./runs.json install_minikube run_benchmark generate_chart -copy ./runs.json "$BUCKET/$RUNTIME-runs.json" -copy ./runs.json "$BUCKET/$(date +'%Y-%m-%d')-$RUNTIME.json" -copy ./daily-chart.png "$BUCKET/$RUNTIME-chart.png" -copy ./weekly-chart.png "$BUCKET/$RUNTIME-weekly-chart.png" +copy ./runs.json "$BUCKET/$DRIVER-$RUNTIME-runs.json" +copy ./runs.json "$BUCKET/$(date +'%Y-%m-%d')-$DRIVER-$RUNTIME.json" +copy ./daily-chart.png "$BUCKET/$DRIVER-$RUNTIME-chart.png" +copy ./weekly-chart.png "$BUCKET/$DRIVER-$RUNTIME-weekly-chart.png" cleanup diff --git a/hack/benchmark/time-to-k8s/public-chart/virtualbox-containerd-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/virtualbox-containerd-benchmark.yaml new file mode 100644 index 000000000000..ed8d46e8e40e --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/virtualbox-containerd-benchmark.yaml @@ -0,0 +1,4 @@ +testcases: + minikube: + setup: minikube start --driver=virtualbox --container-runtime=containerd --memory=max --cpus=max + teardown: minikube delete diff --git a/hack/benchmark/time-to-k8s/public-chart/virtualbox-docker-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/virtualbox-docker-benchmark.yaml new file mode 100644 index 000000000000..e8ba0d61f62d --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/virtualbox-docker-benchmark.yaml @@ -0,0 +1,4 @@ +testcases: + minikube: + setup: minikube start --driver=virtualbox --container-runtime=docker --memory=max --cpus=max + teardown: minikube delete diff --git a/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md b/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md index 950c1d250d8f..62ce29cd9700 100644 --- a/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md +++ b/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md @@ -7,10 +7,18 @@ weight: -99999999 [Benchmarking Machine Specs](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) -## Docker +## Docker driver - Docker runtime -![Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-chart.png) +![Docker Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-docker-chart.png) -## Containerd +## Docker driver - containerd runtime -![Containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/containerd-chart.png) +![Docker containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-containerd-chart.png) + +## VirtualBox driver - Docker runtime + +![VirtualBox Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-docker-chart.png) + +## VirtualBox driver - containerd runtime + +![VirtualBox containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-containerd-chart.png) diff --git a/site/content/en/docs/benchmarks/timeToK8s/weekly_benchmark.md b/site/content/en/docs/benchmarks/timeToK8s/weekly_benchmark.md index 38c81bdd96ed..0ec753eb440f 100644 --- a/site/content/en/docs/benchmarks/timeToK8s/weekly_benchmark.md +++ b/site/content/en/docs/benchmarks/timeToK8s/weekly_benchmark.md @@ -7,10 +7,18 @@ weight: -99999998 [Benchmarking Machine Specs](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) -## Docker +## Docker driver - Docker runtime -![Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-weekly-chart.png) +![Docker Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-docker-weekly-chart.png) -## Containerd +## Docker driver - containerd runtime -![Containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/containerd-weekly-chart.png) +![Docker containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/docker-containerd-weekly-chart.png) + +## VirtualBox driver - Docker runtime + +![VirtualBox Docker Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-docker-weekly-chart.png) + +## VirtualBox driver - containerd runtime + +![VirtualBox containerd Benchmarks](https://time-to-k8s.s3.us-west-1.amazonaws.com/virtualbox-containerd-weekly-chart.png)