More poking of lm-eval-accuracy action

neuralmagic · Apr 15, 2024 · 67843fe · 67843fe
1 parent c46ca5e
commit 67843fe
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/.github/actions/nm-lm-eval-accuracy/action.yml b/.github/actions/nm-lm-eval-accuracy/action.yml
@@ -17,7 +17,7 @@ runs:
       source $(pyenv root)/versions/${{ inputs.python }}/envs/${VENV}/bin/activate
 
       pip3 install git+https://github.com/EleutherAI/lm-evaluation-harness.git@262f879a06aa5de869e5dd951d0ff2cf2f9ba380
-      pip3 install pytest
+      pip3 install pytest openai==1.3.9
 
       SUCCESS=0
       pytest .github/scripts/test_lm_eval_sweep.py -s -v || SUCCESS=$?

diff --git a/.github/scripts/test_lm_eval_sweep.py b/.github/scripts/test_lm_eval_sweep.py
@@ -179,7 +179,13 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 @pytest.mark.parametrize("model_id, eval_def", MODEL_TEST_POINTS)
 def test_lm_eval_correctness(model_id, eval_def):
 
-    vllm_args = ["--model", model_id, "--disable-log-requests"]
+    vllm_args = [
+        "--model",
+        model_id,
+        "--disable-log-requests",
+        "--max_model_len",
+        4096
+    ]
 
     if eval_def.enable_tensor_parallel:
         tp = torch.cuda.device_count()