Varun/benchmark workflow (#28)

Add initial bechmark workflow

---------

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Author: 2 people

.github/actions/nm-benchmark/action.yml

@@ -0,0 +1,17 @@
+name: run vllm benchmarks
+description: 'run vllm benchmarks'
+inputs:
+  output_directory:
+    description: 'output directory to store the benchmark results'
+    required: true
+runs:
+  using: composite
+  steps:
+  - id: benchmark
+    run: |
+      mkdir -p ${{ inputs.output_directory }}
+      SUCCESS=0
+      .github/workflows/scripts/nm-run-benchmarks.sh ${{ inputs.output_directory }} || SUCCESS=$?
+      echo "test=${SUCCESS}" >> "$GITHUB_OUTPUT"
+      exit ${SUCCESS}
+    shell: bash

.github/actions/nm-workflow-info/action.yml

@@ -0,0 +1,25 @@
+name: workflow info
+description: 'give a brief summary of workflow parameters'
+inputs:
+  label:
+    description: "requested runner label (specifies instance)"
+    required: true
+  timeout:
+    description: "time limit for run in minutes "
+    required: true
+  gitref:
+    description: "git commit hash or branch name"
+    required: true
+runs:
+  using: composite
+  steps:
+    - run: |
+        echo "workflow started ..."
+        echo "label: '${{ inputs.label }}'"
+        echo "github actor: '${{ github.actor }}'"
+        echo "repository: '${{ github.repository }}'"
+        echo "gitref: '${{ inputs.gitref }}'"
+        echo "branch name: '${{ github.ref_name }}'"
+        echo "user on instance: $(whoami)"
+        echo "workflow timeout: ${{ inputs.timeout }} (min)"
+      shell: bash

.github/workflows/nm-benchmark.yml

@@ -0,0 +1,72 @@
+name: benchmark 
+on:
+  workflow_dispatch:
+    inputs:
+      label:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      timeout:
+        description: "approximate number of minutes to keep instance up (should be at least 20)."
+        type: string
+        required: true
+      gitref:
+        description: "git commit hash or branch name"
+        type: string
+        required: true
+
+jobs:
+  BENCHMARK:
+
+    runs-on: ${{ inputs.label }}
+    timeout-minutes: ${{ fromJSON(inputs.timeout) }}
+
+    steps:
+      - name: checkout repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gitref }}
+          submodules: recursive
+
+      - name: workflow info
+        uses: ./.github/actions/nm-workflow-info/
+        with:
+          gitref: ${{ inputs.gitref }}
+          label: ${{ inputs.label }}
+          timeout: ${{ inputs.timeout }}
+  
+      # Call the `build` action when available
+      #- name: build
+      #  id: build
+      #  uses: ./.github/actions/build/
+
+      - name: run benchmarks
+        uses: ./.github/actions/nm-benchmark/
+        with:
+          output_directory: benchmark-results
+
+      - name: store benchmark result artifacts
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: ${{ github.run_id }}-${{ inputs.label }}
+          path: benchmark-results
+          retention-days: 90
+
+      #######################################################
+      # TODO (Varun) : Remove pause once things are automated
+      - name: announce pause
+        if: success() || failure()
+        run: |
+          M=${{ inputs.timeout }}
+          R=$((M - 15))
+          S=$((R * 60))
+          echo "pausing for, ${R} minutes"
+
+      - name: pause workflow
+        run: |
+          M=${{ inputs.timeout }}
+          R=$((M - 15))
+          S=$((R * 60))
+          sleep $S

.github/workflows/scripts/nm-run-benchmarks.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+set -u
+  
+if [ $# -ne 1 ];
+then
+  echo "run_benchmarks needs exactly 1 argument - The output path to store the benchmark results"
+  exit -1
+fi
+  
+output_directory=$1
+
+touch $ouptut_directory/bench_test_1.txt
+touch $ouptut_directory/bench_test_2.txt