Fix benchmark code

test/collect_results.py

@@ -40,9 +40,11 @@
         result_dict = {}
         for x in all_results:
             key = (x['model'], x['data_type'])
-            result_dict.setdefault(key, [None] * 4)
-            assert result_dict[key][x['opt_level']] is None
+            result_dict.setdefault(key, [('', '')] * 4)
+            assert result_dict[key][x['opt_level']] == ('', '')
             result_dict[key][x['opt_level']] = (x['pe_toks'], x['tg_toks'])
+        for k in result_dict:
+            assert all(v is not None for v in result_dict[k]), f'result for {k} has None: {result_dict[k]}'
         # prompt evaluation table
         headers[0] = "Prompt Evaluation Performance (Tokens/Sec)"
         value_matrix = [

test/utils.py

@@ -25,11 +25,12 @@ def run_benchmark(ggml_type: str, n_threads: int=1, n_iters: int=1) -> list[str]
 
     @staticmethod
     def run_main(model_path: str, n_threads: int=1, prompt: str='Hello', n_tokens: int=100) -> list[str]:
-        benchmark_options = [
+        options = [
             '-m', model_path, 
             '-t', str(n_threads),
             '-n', str(n_tokens),
-            '-p', prompt
+            '-p', prompt,
+            '--mlock', '--ignore-eos'
         ]
-        run_cmd = ['./src/main'] + benchmark_options
-        return run_cmd
+        run_cmd = ['./src/main'] + options
+        return run_cmd