Fix posting benchmark results and add back Narwhal benchmarks (Mysten…

…Labs#6610)
leina05 · Dec 6, 2022 · 3cded8d · 3cded8d
1 parent 519e115
commit 3cded8d
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 102 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -3,6 +3,7 @@ name: Benchmark
 on:
   push:
      branches: [ main, extensions ]
+  workflow_dispatch:
 
 env:
   CARGO_TERM_COLOR: always
@@ -54,33 +55,63 @@ jobs:
       uses: actions-rs/install@v0.1
       with:
         crate: huniq
-    - name: prepare artifact directory
+    - name: Install python dependencies
+      run: |
+        if [ -f narwhal/benchmark/requirements.txt ]; then pip install -r narwhal/benchmark/requirements.txt; fi
+    - name: Prepare artifact directory
       run: |
         mkdir -p artifacts
-        echo '# Bench results' > artifacts/owned.txt
-        echo '# Bench results' > artifacts/shared.txt
-    - name: compile benchmark
+    - name: Compile benchmark
       uses: actions-rs/cargo@v1
       with:
         command: build
         args: --release
-    - name: run benchmark
+
+    - name: Run benchmarks
       run: |
         set -o pipefail
         cargo run --package sui-benchmark --bin stress -- --log-path /tmp/stress.log --num-client-threads 10 --num-server-threads 24 --num-transfer-accounts 2 bench --target-qps 100 --num-workers 10  --transfer-object 100 --run-duration 60s 2>&1 | huniq | tee -a artifacts/owned.txt
         cargo run --package sui-benchmark --bin stress -- --log-path /tmp/stress.log --num-client-threads 10 --num-server-threads 24 --num-transfer-accounts 2 bench --target-qps 100 --num-workers 10  --shared-counter 100 --run-duration 60s 2>&1 | huniq | tee -a artifacts/shared.txt
-    - name: retrieve benchmark results
+        pushd narwhal/benchmark && fab local | tail -n 31 | tee -a ../../artifacts/narwhal.txt && popd
+
+    - name: Retrieve benchmark results
       id: get-comment-body
       run: |
-        owned="Owned $(cat artifacts/owned.txt | grep -e '|' -e '-' -e 'Bench' -e '+')"
-        shared="Shared $(cat artifacts/shared.txt | grep -e '|' -e '-' -e 'Bench' -e '+')"
-        echo "owned=$owned" >> $GITHUB_OUTPUT
-        echo "shared=$shared" >> $GITHUB_OUTPUT
-    - name: Create commit comment
-      uses: peter-evans/commit-comment@v2
+        delimiter="$(openssl rand -hex 8)"
+
+        owned="$(cat artifacts/owned.txt | grep -e 'Benchmark Report:' -A 1000)"
+        echo "owned<<$delimiter" >> $GITHUB_OUTPUT
+        echo "$owned" >> $GITHUB_OUTPUT
+        echo "$delimiter" >> $GITHUB_OUTPUT
+
+        shared="$(cat artifacts/shared.txt | grep -e 'Benchmark Report:' -A 1000)"
+        echo "shared<<$delimiter" >> $GITHUB_OUTPUT
+        echo "$shared" >> $GITHUB_OUTPUT
+        echo "$delimiter" >> $GITHUB_OUTPUT
+
+        narwhal="$(cat artifacts/narwhal.txt | grep -e 'SUMMARY:' -A 1000)"
+        echo "narwhal<<$delimiter" >> $GITHUB_OUTPUT
+        echo "$narwhal" >> $GITHUB_OUTPUT
+        echo "$delimiter" >> $GITHUB_OUTPUT
+
+    - name: Post commit comment
+      uses: peter-evans/commit-comment@v2.0.1
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         body: |
-          Benchmark Reports:
-          - ${{ steps.get-comment-body.outputs.owned }}
-          - ${{ steps.get-comment-body.outputs.shared }}
+          **Owned Transactions Benchmark Results**
+          ```
+          ${{ steps.get-comment-body.outputs.owned }}
+          ```
+          
+          
+          **Shared Transactions Benchmark Results**
+          ```
+          ${{ steps.get-comment-body.outputs.shared }}
+          ```
+          
+          
+          **Narwhal Benchmark Results**
+          ```
+          ${{ steps.get-comment-body.outputs.narwhal }}
+          ```
diff --git a/narwhal/.github/workflows/bench.yml b/narwhal/.github/workflows/bench.yml
diff --git a/narwhal/benchmark/benchmark/commands.py b/narwhal/benchmark/benchmark/commands.py
@@ -113,4 +113,4 @@ def kill():
     def alias_binaries(origin):
         assert isinstance(origin, str)
         node, client = join(origin, 'narwhal-node'), join(origin, 'narwhal-benchmark-client')
-        return f'rm narwhal-node ; rm narwhal-benchmark-client ; ln -s {node} . ; ln -s {client} .'
+        return f'rm -f narwhal-node ; rm -f narwhal-benchmark-client ; ln -s {node} . ; ln -s {client} .'