From edcf5c6cb1299ca7ad0d467a20012edcca323eb3 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 27 Jun 2024 05:34:37 +0800 Subject: [PATCH 001/325] fix: update example --- run.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/run.sh b/run.sh index 14a9298..1177728 100755 --- a/run.sh +++ b/run.sh @@ -33,8 +33,5 @@ bigcodebench.sanitize --samples $FILE_HEADER.jsonl --calibrate # Check if the ground truth works on your machine bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl -# If the ground truth does not work, you can skip it: -bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl --no-gt - # If the execution is slow: bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl --parallel 32 \ No newline at end of file From d37855f66d352deb7b0888c157595b622e5885d2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 27 Jun 2024 06:04:36 +0800 Subject: [PATCH 002/325] docs: fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 39f58d8..e5a8cb7 100755 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ Reading samples... 1140it [00:00, 1901.64it/s] Evaluating samples... 100%|██████████████████████████████████████████| 1140/1140 [19:53<00:00, 6.75it/s] -BigCodeBench-instruct-calibrated +BigCodeBench-Instruct-calibrated Groundtruth pass rate: 1.000 pass@1: 0.568 ``` From f8b30d29dd32c3f2539cc204376ccb130fcbc0c6 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 00:09:16 +0800 Subject: [PATCH 003/325] fix: remove the trailing dot --- Requirements/requirements-eval.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Requirements/requirements-eval.txt b/Requirements/requirements-eval.txt index 627b910..1d4a48c 100644 --- a/Requirements/requirements-eval.txt +++ b/Requirements/requirements-eval.txt @@ -53,7 +53,7 @@ scikit-image==0.18.0 scikit-learn==1.3.1 scipy==1.7.2 seaborn==0.13.2 -selenium==4.15. +selenium==4.15 sendgrid==6.11.0 shapely==2.0.4 soundfile==0.12.1 From e05de32e4fb5e960fcb656f9a3cd2eb7bca50dd9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 02:26:18 +0800 Subject: [PATCH 004/325] feat: print ou rest for check gt only --- bigcodebench/evaluate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index dc9e7b0..9199e49 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -140,6 +140,10 @@ def evaluate(flags): else: if flags.check_gt_only: + if gt_pass_rate > 0.95: + cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") + else: + cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") return results = { From b0f7948d6427876d5c58644d0cb693bdd03f4944 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 02:59:59 +0800 Subject: [PATCH 005/325] fix: add missing gt res --- bigcodebench/evaluate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 9199e49..05048a0 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -138,8 +138,10 @@ def evaluate(flags): results = compatible_eval_result(results) else: + gt_pass_rate = np.mean([1 if v is not None else 0 for v in expected_time.values()]) if flags.check_gt_only: + if gt_pass_rate > 0.95: cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") else: @@ -244,9 +246,7 @@ def stucking_checker(): mode = "-calibrated" if "sanitized-calibrated" in flags.samples else "" flags.subset = flags.subset[0].upper() + flags.subset[1:] cprint(f"BigCodeBench-{flags.subset}{mode}", "green") - - gt_pass_rate = np.mean([1 if v is not None else 0 for v in expected_time.values()]) - + if flags.no_gt: cprint(f"Groundtruth is not checked", "yellow") else: From ce4831c927b66bb230e0636c1d13a98c836681b8 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 03:22:48 +0800 Subject: [PATCH 006/325] fix: avoid empty data --- bigcodebench/data/bigcodebench.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index aee2c55..d7fea87 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -29,6 +29,8 @@ def _ready_bigcodebench_path(mini=False, noextreme=False, version="default") -> dataset = load_dataset(BIGCODEBENCH_HF, split=BIGCODEBENCH_VERSION) make_cache(url, dataset, path) except: + if os.path.exists(path): + os.remove(path) make_cache(url, None, path, gh=True) return path From a5bff4a7b6b1ce11602841ee7063419f97e64e85 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 03:32:38 +0800 Subject: [PATCH 007/325] fix: increase the gt threshold --- bigcodebench/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 05048a0..37e0a90 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -142,7 +142,7 @@ def evaluate(flags): if flags.check_gt_only: - if gt_pass_rate > 0.95: + if gt_pass_rate > 0.99: cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") else: cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") @@ -250,7 +250,7 @@ def stucking_checker(): if flags.no_gt: cprint(f"Groundtruth is not checked", "yellow") else: - if gt_pass_rate > 0.95: + if gt_pass_rate > 0.99: cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") else: cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") From 20d519af78d8f3d7c38ca2ab7392ee74a013cf8e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 03:46:22 +0800 Subject: [PATCH 008/325] fix: update user permission --- Docker/Evaluate.Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index 5e72687..d0ce0c6 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -18,14 +18,17 @@ RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN cd /bigcodebench && pip install . -RUN pip install -I --timeout 2000 -r https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0/requirements.txt - # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" -RUN chown -R bigcodebenchuser:bigcodebenchuser /bigcodebench -USER bigcodebenchuser +RUN pip install -I --timeout 2000 -r https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0/requirements.txt WORKDIR /app +RUN chown -R bigcodebenchuser:bigcodebenchuser /app + +RUN chmod -R 777 /app + +USER bigcodebenchuser + ENTRYPOINT ["python3", "-m", "bigcodebench.evaluate"] \ No newline at end of file From a1e19cd285eb49b0b01dd3fefb147a375dd74987 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 04:20:56 +0800 Subject: [PATCH 009/325] fix: correctly reflect gt res --- bigcodebench/gen/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/gen/util/__init__.py b/bigcodebench/gen/util/__init__.py index 79a2a78..0f6f28d 100644 --- a/bigcodebench/gen/util/__init__.py +++ b/bigcodebench/gen/util/__init__.py @@ -110,7 +110,7 @@ def trusted_check( time.sleep(0.1) if times.value == -1: - times = -1 + times = None else: times = times.value From 1702f0f0faca68f00152500300ecd3a9ecd5d150 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 04:43:52 +0800 Subject: [PATCH 010/325] fix: cast the ram limit to int --- bigcodebench/evaluate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 37e0a90..b761530 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -286,9 +286,9 @@ def main(): parser.add_argument("--samples", required=True, type=str) parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) - parser.add_argument("--max-as-limit", default=128*1024, type=float) - parser.add_argument("--max-data-limit", default=4*1024, type=float) - parser.add_argument("--max-stack-limit", default=5, type=float) + parser.add_argument("--max-as-limit", default=128*1024, type=int) + parser.add_argument("--max-data-limit", default=4*1024, type=int) + parser.add_argument("--max-stack-limit", default=5, type=int) parser.add_argument( "--check-gt-only", action="store_true", help="Check the groundtruth" ) From c0ad781177e84f4ba2b9b6107ab60ac877185a2c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 04:45:25 +0800 Subject: [PATCH 011/325] fix: init times as -1 --- bigcodebench/gen/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/gen/util/__init__.py b/bigcodebench/gen/util/__init__.py index 0f6f28d..6de3468 100644 --- a/bigcodebench/gen/util/__init__.py +++ b/bigcodebench/gen/util/__init__.py @@ -85,7 +85,7 @@ def trusted_check( ): timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", 120) + 1 # shared memory objects - times = Value("d") + times = Value("d", -1) manager = Manager() p = multiprocessing.Process( From 7c36dd7617b92d983b4a987b35338164e5fdbd25 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 05:20:50 +0800 Subject: [PATCH 012/325] docs: update known issues --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e5a8cb7..34a7ab6 100755 --- a/README.md +++ b/README.md @@ -329,11 +329,13 @@ python get_results.py ## 💻 LLM-generated Code We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): -* See the attachment of our [v0.1.2](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. +* See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. ## Known Issues -- [ ] Due to the flakes in the evaluation, the execution results may vary slightly (~0.1%) between runs. We are working on improving the evaluation stability. +- [ ] Due to the flakes in the evaluation, the execution results may vary slightly (~0.2%) between runs. We are working on improving the evaluation stability. + +- [ ] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. - [ ] We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code. From afbf8debf199102b20c81afa683016822ff56589 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 28 Jun 2024 06:06:46 +0800 Subject: [PATCH 013/325] fix: typo in generate --- bigcodebench/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index ba1f9f9..7edebfc 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -34,7 +34,7 @@ def codegen( dataset = get_bigcodebench() if model.is_direct_completion() and subset == "instruct": - raise Exception("Base model does not support direct completion for instructode tasks") + raise Exception("Base model does not support direct completion for instruct tasks") # create save_path if it doesn't exist, e.g., a/b.jsonl dirname = os.path.dirname(save_path) From bca185919d1921422bb749794f12866a74603909 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 29 Jun 2024 09:34:52 +0000 Subject: [PATCH 014/325] remove results --- analysis/results.json | 783 ------------------------------------------ 1 file changed, 783 deletions(-) delete mode 100644 analysis/results.json diff --git a/analysis/results.json b/analysis/results.json deleted file mode 100644 index 3276bb7..0000000 --- a/analysis/results.json +++ /dev/null @@ -1,783 +0,0 @@ -{ - "Magicoder-S-DS-6.7B": { - "link": "https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B", - "open-data": "Partial", - "pass@1": { - "instruct": 36.2, - "complete": 47.6 - }, - "prompted": true, - "size": 6.7, - "lazy": false - }, - "StarCoder2-15B-Instruct-v0.1": { - "link": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1", - "open-data": "Full", - "pass@1": { - "instruct": 37.6, - "complete": 45.1 - }, - "prompted": true, - "size": 15, - "lazy": true - }, - "StarCoder2-3B": { - "link": "https://huggingface.co/bigcode/starcoder2-3b", - "open-data": "Full", - "pass@1": { - "instruct": null, - "complete": 21.4 - }, - "prompted": false, - "size": 3, - "lazy": false - }, - "StarCoder2-7B": { - "link": "https://huggingface.co/bigcode/starcoder2-7b", - "open-data": "Full", - "pass@1": { - "instruct": null, - "complete": 27.7 - }, - "prompted": false, - "size": 7, - "lazy": false - }, - "StarCoder2-15B": { - "link": "https://huggingface.co/bigcode/starcoder2-15b", - "open-data": "Full", - "pass@1": { - "instruct": null, - "complete": 38.4 - }, - "prompted": false, - "size": 15, - "lazy": false - }, - "CodeQwen1.5-7B": { - "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 45.6 - }, - "prompted": false, - "size": 7, - "lazy": false - }, - "CodeGemma-2B": { - "link": "https://huggingface.co/google/codegemma-2b", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 23.9 - }, - "prompted": false, - "size": 2, - "lazy": false - }, - "CodeGemma-7B": { - "link": "https://huggingface.co/google/codegemma-7b", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 38.3 - }, - "prompted": false, - "size": 7, - "lazy": false - }, - "CodeGemma-7B-Instruct": { - "link": "https://huggingface.co/google/codegemma-7b-it", - "open-data": "None", - "pass@1": { - "instruct": 32.3, - "complete": 39.3 - }, - "prompted": true, - "size": 7, - "lazy": false - }, - "GPT-3.5-Turbo-0125": { - "link": "https://openai.com/index/new-embedding-models-and-api-updates", - "open-data": "None", - "pass@1": { - "instruct": 39.1, - "complete": 50.6 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "GPT-4o-2024-05-13": { - "link": "https://openai.com/index/hello-gpt-4o/", - "open-data": "None", - "pass@1": { - "instruct": 51.1, - "complete": 61.1 - }, - "prompted": true, - "size": null, - "lazy": true - }, - "GPT-4-Turbo-2024-04-09": { - "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday", - "open-data": "None", - "pass@1": { - "instruct": 48.2, - "complete": 58.2 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "GPT-4-0613": { - "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday", - "open-data": "None", - "pass@1": { - "instruct": 46.0, - "complete": 57.2 - }, - "prompted": true, - "size": null, - "lazy": true - }, - "CodeLlama-7B-Base": { - "link": "https://huggingface.co/codellama/CodeLlama-7b-hf", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 28.7 - }, - "prompted": false, - "size": 7, - "lazy": false - }, - "CodeLlama-13B-Base": { - "link": "https://huggingface.co/codellama/CodeLlama-13b-hf", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 32.0 - }, - "prompted": false, - "size": 13, - "lazy": false - }, - "CodeLlama-7B-Instruct": { - "link": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf", - "open-data": "None", - "pass@1": { - "instruct": 21.9, - "complete": 25.7 - }, - "prompted": true, - "size": 7, - "lazy": false - }, - "CodeLlama-13B-Instruct": { - "link": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf", - "open-data": "None", - "pass@1": { - "instruct": 28.5, - "complete": 31.7 - }, - "prompted": true, - "size": 13, - "lazy": true - }, - "Mistral-Large-2402": { - "link": "https://mistral.ai/news/mistral-large/", - "open-data": "None", - "pass@1": { - "instruct": 30.0, - "complete": 38.3 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "Mistral-Small-2402": { - "link": "https://mistral.ai/news/mistral-large/", - "open-data": "None", - "pass@1": { - "instruct": 32.1, - "complete": 41.3 - }, - "prompted": true, - "size": null, - "lazy": true - }, - "Mixtral-8x22B-Base": { - "link": "https://huggingface.co/mistralai/Mixtral-8x22B-v0.1", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 45.3 - }, - "prompted": false, - "size": 44, - "lazy": false - }, - "Mixtral-8x22B-Instruct": { - "link": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1", - "open-data": "None", - "pass@1": { - "instruct": 40.6, - "complete": 50.2 - }, - "prompted": true, - "size": 44, - "lazy": false - }, - "CodeLlama-34B-Base": { - "link": "https://huggingface.co/codellama/CodeLlama-34b-hf", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 37.1 - }, - "prompted": false, - "size": 34, - "lazy": false - }, - "CodeLlama-34B-Instruct": { - "link": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf", - "open-data": "None", - "pass@1": { - "instruct": 29.0, - "complete": 35.6 - }, - "prompted": true, - "size": 34, - "lazy": false - }, - "CodeLlama-70B-Base": { - "link": "https://huggingface.co/codellama/CodeLlama-70b-hf", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 44.0 - }, - "prompted": false, - "size": 70, - "lazy": false - }, - "CodeLlama-70B-Instruct": { - "link": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf", - "open-data": "None", - "pass@1": { - "instruct": 40.7, - "complete": 49.6 - }, - "prompted": true, - "size": 70, - "lazy": false - }, - "CodeQwen1.5-7B-Chat": { - "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 39.6, - "complete": 43.6 - }, - "prompted": true, - "size": 7, - "lazy": false - }, - "Qwen1.5-110B-Chat": { - "link": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 35.0, - "complete": 44.4 - }, - "prompted": true, - "size": 110, - "lazy": true - }, - "Qwen1.5-72B-Chat": { - "link": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 33.2, - "complete": 40.3 - }, - "prompted": true, - "size": 72, - "lazy": false - }, - "Qwen1.5-32B-Chat": { - "link": "https://huggingface.co/Qwen/Qwen1.5-32B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 32.3, - "complete": 42.0 - }, - "prompted": true, - "size": 32, - "lazy": false - }, - "DeepSeek-V2-Chat": { - "link": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat", - "open-data": "None", - "pass@1": { - "instruct": 40.4, - "complete": 49.0 - }, - "prompted": true, - "size": 21, - "lazy": false - }, - "DeepSeek-Coder-1.3B-Base": { - "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 22.2 - }, - "prompted": false, - "size": 1.3, - "lazy": false - }, - "DeepSeek-Coder-1.3B-Instruct": { - "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct", - "open-data": "None", - "pass@1": { - "instruct": 22.8, - "complete": 29.6 - }, - "prompted": true, - "size": 1.3, - "lazy": false - }, - "DeepSeek-Coder-33B-Base": { - "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 46.6 - }, - "prompted": false, - "size": 33, - "lazy": false - }, - "DeepSeek-Coder-33B-Instruct": { - "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct", - "open-data": "None", - "pass@1": { - "instruct": 42.0, - "complete": 51.1 - }, - "prompted": true, - "size": 33, - "lazy": false - }, - "DeepSeek-Coder-6.7B-Base": { - "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 41.8 - }, - "prompted": false, - "size": 6.7, - "lazy": false - }, - "DeepSeek-Coder-6.7B-Instruct": { - "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct", - "open-data": "None", - "pass@1": { - "instruct": 35.5, - "complete": 43.8 - }, - "prompted": true, - "size": 6.7, - "lazy": false - }, - "Llama-3-70B-Base": { - "link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 43.3 - }, - "prompted": false, - "size": 70, - "lazy": false - }, - "Llama-3-70B-Instruct": { - "link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct", - "open-data": "None", - "pass@1": { - "instruct": 43.6, - "complete": 54.5 - }, - "prompted": true, - "size": 70, - "lazy": false - }, - "Llama-3-8B-Base": { - "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 28.8 - }, - "prompted": false, - "size": 8, - "lazy": false - }, - "Llama-3-8B-Instruct": { - "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct", - "open-data": "None", - "pass@1": { - "instruct": 31.9, - "complete": 36.9 - }, - "prompted": true, - "size": 8, - "lazy": false - }, - "Granite-Code-3B-Instruct": { - "link": "https://huggingface.co/ibm-granite/granite-3b-code-instruct", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 31.4 - }, - "prompted": true, - "size": 3, - "lazy": false - }, - "Granite-Code-8B-Instruct": { - "link": "https://huggingface.co/ibm-granite/granite-8b-code-instruct", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 39.7 - }, - "prompted": true, - "size": 8, - "lazy": false - }, - "Granite-Code-20B-Instruct": { - "link": "https://huggingface.co/ibm-granite/granite-20b-code-instruct", - "open-data": "None", - "pass@1": { - "instruct": 34.0, - "complete": 42.0 - }, - "prompted": true, - "size": 20, - "lazy": false - }, - "Granite-Code-34B-Instruct": { - "link": "https://huggingface.co/ibm-granite/granite-34b-code-instruct", - "open-data": "None", - "pass@1": { - "instruct": 36.1, - "complete": 44.4 - }, - "prompted": true, - "size": 34, - "lazy": false - }, - "Granite-Code-3B-Base": { - "link": "https://huggingface.co/ibm-granite/granite-3b-code-base", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 20.0 - }, - "prompted": false, - "size": 3, - "lazy": false - }, - "Granite-Code-8B-Base": { - "link": "https://huggingface.co/ibm-granite/granite-8b-code-base", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 35.7 - }, - "prompted": false, - "size": 8, - "lazy": false - }, - "Granite-Code-20B-Base": { - "link": "https://huggingface.co/ibm-granite/granite-20b-code-base", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 25.6 - }, - "prompted": false, - "size": 20, - "lazy": false - }, - "Granite-Code-34B-Base": { - "link": "https://huggingface.co/ibm-granite/granite-34b-code-base", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 38.4 - }, - "prompted": false, - "size": 34, - "lazy": false - }, - "Claude-3-Haiku-20240307": { - "link": "https://www.anthropic.com/news/claude-3-family", - "open-data": "None", - "pass@1": { - "instruct": 39.4, - "complete": 50.1 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "Claude-3-Sonnet-20240229": { - "link": "https://www.anthropic.com/news/claude-3-family", - "open-data": "None", - "pass@1": { - "instruct": 42.7, - "complete": 53.8 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "Claude-3-Opus-20240229": { - "link": "https://www.anthropic.com/news/claude-3-family", - "open-data": "None", - "pass@1": { - "instruct": 45.5, - "complete": 57.4 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "Yi-1.5-34B-Chat": { - "link": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 33.9, - "complete": 43.8 - }, - "prompted": true, - "size": 34, - "lazy": false - }, - "Yi-1.5-34B": { - "link": "https://huggingface.co/01-ai/Yi-1.5-34B", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 40.3 - }, - "prompted": false, - "size": 34, - "lazy": false - }, - "Yi-1.5-9B-Chat": { - "link": "https://huggingface.co/01-ai/Yi-1.5-9B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 34.5, - "complete": 42.4 - }, - "prompted": true, - "size": 9, - "lazy": false - }, - "Yi-1.5-9B": { - "link": "https://huggingface.co/01-ai/Yi-1.5-9B", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 35.5 - }, - "prompted": false, - "size": 9, - "lazy": false - }, - "Yi-1.5-6B-Chat": { - "link": "https://huggingface.co/01-ai/Yi-1.5-6B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 25.6, - "complete": 33.9 - }, - "prompted": true, - "size": 9, - "lazy": false - }, - "Yi-1.5-6B": { - "link": "https://huggingface.co/01-ai/Yi-1.5-6B", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 27.4 - }, - "prompted": false, - "size": 9, - "lazy": false - }, - "Qwen2-57B-A14B": { - "link": "https://huggingface.co/Qwen/Qwen2-57B-A14B", - "open-data": "None", - "pass@1": { - "instruct": 36.1, - "complete": 46.8 - }, - "prompted": true, - "size": 14, - "lazy": false - }, - "Qwen2-7B-Instruct": { - "link": "https://huggingface.co/Qwen/Qwen2-7B-Instruct", - "open-data": "None", - "pass@1": { - "instruct": 29.1, - "complete": 42.1 - }, - "prompted": true, - "size": 7, - "lazy": true - }, - "Qwen2-72B-Chat": { - "link": "https://huggingface.co/Qwen/Qwen2-72B-Chat", - "open-data": "None", - "pass@1": { - "instruct": 38.5, - "complete": 54.0 - }, - "prompted": true, - "size": 72, - "lazy": false - }, - "Gemini-1.5-Pro-API-0514": { - "link": "https://deepmind.google/technologies/gemini/pro", - "open-data": "None", - "pass@1": { - "instruct": 43.8, - "complete": 57.5 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "Gemini-1.5-Flash-API-0514": { - "link": "https://deepmind.google/technologies/gemini/flash/", - "open-data": "None", - "pass@1": { - "instruct": 43.5, - "complete": 55.1 - }, - "prompted": true, - "size": null, - "lazy": false - }, - "OpenCodeInterpreter-DS-33B": { - "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B", - "open-data": "Partial", - "pass@1": { - "instruct": null, - "complete": 51.0 - }, - "prompted": false, - "size": 33, - "lazy": false - }, - "OpenCodeInterpreter-DS-6.7B": { - "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B", - "open-data": "Partial", - "pass@1": { - "instruct": null, - "complete": 44.6 - }, - "prompted": false, - "size": 6.7, - "lazy": false - }, - "OpenCodeInterpreter-DS-1.3B": { - "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B", - "open-data": "Partial", - "pass@1": { - "instruct": null, - "complete": 29.8 - }, - "prompted": false, - "size": 1.3, - "lazy": false - }, - "Phi-3-medium-128k-instruct": { - "link": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct", - "open-data": "None", - "pass@1": { - "instruct": 37.6, - "complete": 48.7 - }, - "prompted": true, - "size": 14, - "lazy": false - }, - "Phi-3-small-128k-instruct": { - "link": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 39.4 - }, - "prompted": true, - "size": 7, - "lazy": false - }, - "Codestral-22B-v0.1": { - "link": "https://huggingface.co/mistralai/Codestral-22B-v0.1", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 52.8 - }, - "prompted": false, - "size": 22, - "lazy": false - }, - "Mistral-7B-Instruct-v0.3": { - "link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3", - "open-data": "None", - "pass@1": { - "instruct": 19.5, - "complete": 27.3 - }, - "prompted": true, - "size": 7, - "lazy": false - }, - "Mistral-7B-v0.3": { - "link": "https://huggingface.co/mistralai/Mistral-7B-v0.3", - "open-data": "None", - "pass@1": { - "instruct": null, - "complete": 23.5 - }, - "prompted": false, - "size": 7, - "lazy": false - }, - "Command R+": { - "link": "https://huggingface.co/CohereForAI/c4ai-command-r-plus", - "open-data": "None", - "pass@1": { - "instruct": 33.8, - "complete": 41.9 - }, - "prompted": true, - "size": 104, - "lazy": false - } -} \ No newline at end of file From 4da429c49a3c3bd8be4d1ea35555bfa2e2ac1aa8 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 29 Jun 2024 09:37:19 +0000 Subject: [PATCH 015/325] feat: collect eval results --- analysis/get_results.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index c3fbb45..867cdef 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -144,6 +144,7 @@ def split_gen(): def read_task_perf(task="complete"): model_results = dict() + result_files = [] for model, info in model_info.items(): if task == "instruct" and (not info["prompted"] or info["name"] in ["Granite-Code-3B-Instruct", "Granite-Code-8B-Instruct"]): continue @@ -164,13 +165,14 @@ def read_task_perf(task="complete"): except: continue + result_files.append(file) with open(file, "r") as f: data = json.load(f) for task_id, perfs in data["eval"].items(): status = 1 if perfs[0]["status"] == "pass" else 0 task_perf[task_id] = status model_results[info["name"]] = task_perf - return model_results + return model_results, result_files def get_winner_df(data_dict, task, task_level=True, no_tie=True): @@ -313,8 +315,16 @@ def push_ds(ds, path, local=False): model_info = update_model_info(model_info) results = get_results() - complete_data = read_task_perf("complete") - instruct_data = read_task_perf("instruct") + files = [] + complete_data, complete_files = read_task_perf("complete") + instruct_data, instruct_files = read_task_perf("instruct") + files.extend(complete_files) + files.extend(instruct_files) + shutil.rmtree("eval_results", ignore_errors=True) + os.makedirs("eval_results", exist_ok=True) + for file in files: + shutil.copy(file, "eval_results") + complete_solve_rate = get_solve_rate(complete_data, task="complete") instruct_solve_rate = get_solve_rate(instruct_data, task="instruct") solve_rate_ds = DatasetDict({"complete": complete_solve_rate, "instruct": instruct_solve_rate}) From f7d3cc871d8da329a162b639feb7ad392c837444 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 29 Jun 2024 09:38:52 +0000 Subject: [PATCH 016/325] feat: add wavecoder and gemma2 9 it --- analysis/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index fc4e118..3bdc8a3 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -531,4 +531,18 @@ "size": 70, "open-data": "None", }, + "microsoft/wavecoder-ultra-6.7b": { + "name": "WaveCoder-Ultra-6.7B", + "link": "https://huggingface.co/microsoft/wavecoder-ultra-6.7b", + "prompted": True, + "size": 6.7, + "open-data": "None", + }, + "google/gemma-2-9b-it": { + "name": "Gemma-2-9B-Instruct", + "link": "https://huggingface.co/google/gemma-2-9b-it", + "prompted": True, + "size": 9, + "open-data": "None", + }, } \ No newline at end of file From 4cc80a347d6929a4417116a6cb30ad573008973c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 29 Jun 2024 17:46:28 +0800 Subject: [PATCH 017/325] fix: avoid undeclared gt res --- bigcodebench/evaluate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index b761530..0477bcb 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -131,6 +131,8 @@ def evaluate(flags): else: expected_time = {task_id: None for task_id in problems} + gt_pass_rate = np.mean([1 if v is not None else 0 for v in expected_time.values()]) + if os.path.isfile(result_path): print(f"Load from previous results from {result_path}") with open(result_path, "r") as f: @@ -138,7 +140,6 @@ def evaluate(flags): results = compatible_eval_result(results) else: - gt_pass_rate = np.mean([1 if v is not None else 0 for v in expected_time.values()]) if flags.check_gt_only: From 44ac28f1439d95b811c6e0a4ed06c6ec4b574c35 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 29 Jun 2024 17:47:24 +0800 Subject: [PATCH 018/325] fix: remove extra newline --- bigcodebench/evaluate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 0477bcb..619be0e 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -140,7 +140,6 @@ def evaluate(flags): results = compatible_eval_result(results) else: - if flags.check_gt_only: if gt_pass_rate > 0.99: From fc0588d257aa9f89d030699b2a382f83b6976532 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 01:24:43 +0800 Subject: [PATCH 019/325] fix: temp fix for cleaning --- Docker/Evaluate.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index d0ce0c6..ea3fc6d 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -31,4 +31,6 @@ RUN chmod -R 777 /app USER bigcodebenchuser -ENTRYPOINT ["python3", "-m", "bigcodebench.evaluate"] \ No newline at end of file +ENTRYPOINT ["python3", "-m", "bigcodebench.evaluate"] + +CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep '^ *[0-9]\\+ bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs kill; fi; rm -rf /tmp/*"] \ No newline at end of file From b611e49847367fc058731928e1143fff815a1970 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 01:27:49 +0800 Subject: [PATCH 020/325] fix: add -r for kill --- Docker/Evaluate.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index ea3fc6d..ca0a407 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -33,4 +33,4 @@ USER bigcodebenchuser ENTRYPOINT ["python3", "-m", "bigcodebench.evaluate"] -CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep '^ *[0-9]\\+ bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs kill; fi; rm -rf /tmp/*"] \ No newline at end of file +CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep '^ *[0-9]\\+ bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/*"] \ No newline at end of file From 8b57962ac5bb73f1e2f810644a5c22de1583652d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 01:31:03 +0800 Subject: [PATCH 021/325] docs: add eval clean --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 34a7ab6..727d46c 100755 --- a/README.md +++ b/README.md @@ -249,6 +249,10 @@ Then, run the evaluation: bigcodebench.evaluate --subset [complete|instruct] --samples samples-sanitized-calibrated.jsonl # ...If you really don't want to check the ground truths bigcodebench.evaluate --subset [complete|instruct] --samples samples-sanitized-calibrated.jsonl --no-gt + +# You are strongly recommended to use the following command to clean up the environment after evaluation: +pids=$(ps -u $(id -u) -o pid,comm | grep '^ *[0-9]\\+ bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; +rm -rf /tmp/* ``` > [!Tip] From e4fc2f4a85ade3e693d9bf3d6cfdebf7f2aa0833 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 02:40:10 +0800 Subject: [PATCH 022/325] docs: add links and citation --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 727d46c..6bebc00 100755 --- a/README.md +++ b/README.md @@ -4,7 +4,12 @@

+ + + + + @@ -347,10 +352,10 @@ We share pre-generated code samples from LLMs we have [evaluated](https://huggin ```bibtex @article{zhuo2024bigcodebench, - title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}, - author={Terry Yue Zhuo and Minh Chien Vu and Jenny Chim and Han Hu and Wenhao Yu and Ratnadira Widyasari and Imam Nur Bani Yusuf and Haolan Zhan and Junda He and Indraneil Paul and Simon Brunner and Chen Gong and Thong Hoang and Armel Randy Zebaze and Xiaoheng Hong and Wen-Ding Li and Jean Kaddour and Ming Xu and Zhihan Zhang and Prateek Yadav and Naman Jain and Alex Gu and Zhoujun Cheng and Jiawei Liu and Qian Liu and Zijian Wang and David Lo and Binyuan Hui and Niklas Muennighoff and Daniel Fried and Xiaoning Du and Harm de Vries and Leandro Von Werra}, - journal={arXiv preprint arXiv:2406.15877}, - year={2024} + title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}, + author={Zhuo, Terry Yue and Vu, Minh Chien and Chim, Jenny and Hu, Han and Yu, Wenhao and Widyasari, Ratnadira and Yusuf, Imam Nur Bani and Zhan, Haolan and He, Junda and Paul, Indraneil and others}, + journal={arXiv preprint arXiv:2406.15877}, + year={2024} } ``` From 67cac6ccf4ede7a6c4dc2c08312fe1e11a1c04d3 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 02:43:33 +0800 Subject: [PATCH 023/325] docs: swap links --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6bebc00..d2749de 100755 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ + -

From c3f79159e3120fcd55f956fa927d209b32429998 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 29 Jun 2024 18:54:23 +0000 Subject: [PATCH 024/325] docs: update badge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d2749de..3862d3d 100755 --- a/README.md +++ b/README.md @@ -5,14 +5,14 @@

- + - +

From e5195ca204c82803b887638beb68a935e9e5e5cf Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 03:57:28 +0800 Subject: [PATCH 025/325] docs: update paper link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3862d3d..33c2330 100755 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ - + From 59ae6741eaff19b4bb5cc9a211c5f20720c77c2c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 19:02:04 +0800 Subject: [PATCH 026/325] docs: swap badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33c2330..7be8693 100755 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ + -

From 405890f71ea9a38ed052a53bff5ffc4186fe0727 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 19:16:06 +0800 Subject: [PATCH 027/325] add autocoders --- analysis/utils.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index 3bdc8a3..6ae5bcf 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -545,4 +545,25 @@ "size": 9, "open-data": "None", }, + "Bin12345/AutoCoder": { + "name": "AutoCoder", + "link": "https://huggingface.co/Bin12345/AutoCoder", + "prompted": True, + "size": 33, + "open-data": "None", + }, + "Bin12345/AutoCoder_S_6.7B": { + "name": "AutoCoder-S-6.7B", + "link": "https://huggingface.co/Bin12345/AutoCoder_S_6.7B", + "prompted": True, + "size": 6.7, + "open-data": "None", + }, + "Bin12345/AutoCoder_QW_7B": { + "name": "AutoCoder-QW-7B", + "link": "https://huggingface.co/Bin12345/AutoCoder_QW_7B", + "prompted": True, + "size": 7, + "open-data": "None", + }, } \ No newline at end of file From 84f399c373cadfd28416d5918c4e2167bddab2ac Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 19:16:27 +0800 Subject: [PATCH 028/325] fix: rm txt writing --- analysis/get_results.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index 867cdef..dfa9f56 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -269,9 +269,6 @@ def get_solve_rate(data_dict, task="complete"): for task_id in range(1140): task_solve_count[f"BigCodeBench/{task_id}"].append(task_perf[f"BigCodeBench/{task_id}"]) solve_rate = {task_id: round(np.mean(perfs) * 100, 1) for task_id, perfs in task_solve_count.items()} - with open(f"{task}_solve_rate.txt", "w") as f: - f.write(f"Number of unsolved tasks: {sum([1 for task_id, solve_rate in solve_rate.items() if solve_rate == 0])}\n") - f.write(f"Number of fully solved tasks: {sum([1 for task_id, solve_rate in solve_rate.items() if solve_rate == 100])}\n") return Dataset.from_dict({"task_id": list(solve_rate.keys()), "solve_rate": list(solve_rate.values())}) From 3ed7502a62e2f39292c17abebf5561bdf6555802 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 30 Jun 2024 15:23:56 +0000 Subject: [PATCH 029/325] add more models --- analysis/utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index 6ae5bcf..65e8710 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -566,4 +566,32 @@ "size": 7, "open-data": "None", }, + "SenseLLM/ReflectionCoder-DS-33B": { + "name": "ReflectionCoder-DS-33B", + "link": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-33B", + "prompted": True, + "size": 33, + "open-data": "Partial", + }, + "SenseLLM/ReflectionCoder-DS-6.7B": { + "name": "ReflectionCoder-DS-6.7B", + "link": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-6.7B", + "prompted": True, + "size": 6.7, + "open-data": "Partial", + }, + "SenseLLM/ReflectionCoder-CL-34B": { + "name": "ReflectionCoder-CL-34B", + "link": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-34B", + "prompted": True, + "size": 34, + "open-data": "Partial", + }, + "SenseLLM/ReflectionCoder-CL-7B": { + "name": "ReflectionCoder-CL-7B", + "link": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-7B", + "prompted": True, + "size": 7, + "open-data": "Partial", + }, } \ No newline at end of file From a02256ff12cd8a30e9b87de5ebb5e7804010d228 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 2 Jul 2024 03:11:04 +0800 Subject: [PATCH 030/325] docs: update links --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7be8693..241116b 100755 --- a/README.md +++ b/README.md @@ -18,16 +18,16 @@

🌸About • 🔥Quick Start • - 💻LLM code • - 🔍Failure inspection • + 🔍Failure Inspection • 🚀Full Script • 📊Result Analysis • - 🐞Known issues • + 💻LLM-generated Code • + 🐞Known Issues • 📜Citation • 🙏Acknowledgement

-## About +## 🌸 About ### BigCodeBench @@ -307,7 +307,7 @@ Here are some tips to speed up the evaluation: -## Failure Inspection +## 🔍 Failure Inspection You can inspect the failed samples by using the following command: @@ -315,7 +315,7 @@ You can inspect the failed samples by using the following command: bigcodebench.inspect --eval-results sample-sanitized-calibrated_eval_results.json --in-place ``` -## Full Script +## 🚀 Full Script We provide a sample script to run the full pipeline: @@ -323,7 +323,7 @@ We provide a sample script to run the full pipeline: bash run.sh ``` -## Result Analysis +## 📊 Result Analysis We provide a script to replicate the analysis like Elo Rating and Task Solve Rate, which helps you understand the performance of the models further. @@ -340,7 +340,7 @@ python get_results.py We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): * See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. -## Known Issues +## 🐞 Known Issues - [ ] Due to the flakes in the evaluation, the execution results may vary slightly (~0.2%) between runs. We are working on improving the evaluation stability. From c1422e2c2feb0753895d28a5beced62135ee2bc6 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 3 Jul 2024 03:13:56 +0800 Subject: [PATCH 031/325] fix: only save valid gt time --- bigcodebench/evaluate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 619be0e..6e94672 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -72,8 +72,9 @@ def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit, print(f"Expected outputs computed in {time.time() - tbegin:.2f}s") - with open(cache_file, "wb") as f: - pickle.dump(expected_time, f) + if any(expected_time.values()): + with open(cache_file, "wb") as f: + pickle.dump(expected_time, f) return expected_time From 60c752bc1e609f08a7af0892b7f15618c75b36a8 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 4 Jul 2024 06:23:25 +0800 Subject: [PATCH 032/325] feat: add hard script --- analysis/bcb_hard.py | 128 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 analysis/bcb_hard.py diff --git a/analysis/bcb_hard.py b/analysis/bcb_hard.py new file mode 100644 index 0000000..e12c3b1 --- /dev/null +++ b/analysis/bcb_hard.py @@ -0,0 +1,128 @@ +import pickle +import json +import numpy as np +from tqdm import tqdm +from ast import literal_eval +from glob import glob +from sentence_transformers import SentenceTransformer, util +import matplotlib.pyplot as plt +from datasets import load_dataset, Dataset, Features, Value, Sequence + +from utils import * + +def embed_sentences(data, col_name, id_name, model, save_path, push_to_hub=False): + pool = model.start_multi_process_pool() + embeddings = model.encode_multi_process(data[col_name], pool=pool) + qids = data[id_name] + features = Features({id_name: Value(dtype='string'), 'embeddings': Sequence(Value('float32'))}) + embed_dict = { + id_name: qids, + "embeddings": embeddings + } + embed_ds = Dataset.from_dict(embed_dict, features=features) + if push_to_hub: + embed_ds.push_to_hub(f"bigcode/{save_path}") + else: + embed_ds.save_to_disk(save_path) + return embed_ds + + +def get_top_docs(query_embs, doc_emb, docs): + scores = np.dot(query_embs, doc_emb.T) + top_doc_indices = np.argmax(scores, axis=1) + top_scores = scores[np.arange(len(scores)), top_doc_indices] + results = [(i, docs[doc_idx], score) for i, (doc_idx, score) in tqdm(enumerate(zip(top_doc_indices, top_scores)))] + + return results + + +def filter_top_k_percent(results, k_percent): + all_scores = [score for _, score in results] + threshold = np.percentile(all_scores, 100 - k_percent) + filtered_results = [(i, doc, score) for i, doc, score in results if score > threshold] + return filtered_results + + +def filter_top_threshold(results, threshold): + filtered_results = [(i, doc, score) for i, doc, score in results if score > threshold] + return filtered_results + + +def read_task_perf(top_tid, task="complete"): + model_results = dict() + result_files = [] + for model, info in model_info.items(): + if task == "instruct" and (not info["prompted"] or info["name"] in ["Granite-Code-3B-Instruct", "Granite-Code-8B-Instruct"]): + continue + task_perf = {f"BigCodeBench/{task_id}": 0 for task_id in range(1140)} + model = model.replace("/", "--") + if info["link"].startswith("https://huggingface.co/"): + model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") + try: + if info["prompted"]: + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") + if files: + file = files[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + except: + continue + with open(file, "r") as f: + data = json.load(f) + for task_id, perfs in data["eval"].items(): + status = 1 if perfs[0]["status"] == "pass" else 0 + task_perf[task_id] = status + model_results[info["name"]] = np.mean([status for tid, status in task_perf.items() if tid in top_tid]) + return sorted(model_results.items(), key=lambda x: x[1], reverse=True) + + +if __name__ == "__main__": + bcb = load_dataset("bigcode/bigcodebench", trust_remote_code=True, split="v0.1.0_hf") + se = load_dataset("bigcode/stack-exchange-preferences-20230914-clean-anonymization", trust_remote_code=True, split="train") + model = SentenceTransformer("sentence-transformers/all-mpnet-base-v2") + + se_embed = embed_sentences(se, "question", "qid", model, "stack-exchange-embeddings-20230914", push_to_hub=True) + bcb_embed = embed_sentences(bcb, "complete_prompt", "task_id", model, "bigcodebench-doc-embeddings", push_to_hub=True) + + solve_rate = load_dataset("bigcode/bigcodebench-solve-rate", trust_remote_code=True, split="complete") + + query_embs = np.array(se_embed["embeddings"]) + doc_emb = np.array(bcb_embed["embeddings"]) + docs = bcb_embed["task_id"] + retrieval_results = get_top_docs(query_embs, doc_emb, docs) + + Dataset.from_dict({"qid": [i for i, _, _ in retrieval_results], "tid": [doc for _, doc, _ in retrieval_results], "score": [score for _, _, score in retrieval_results]}).push_to_hub("bigcode/se_bcb_results") + + retrieval_ds = load_dataset("bigcode/se_bcb_results", trust_remote_code=True, split="train") + retrieval_ds = load_dataset("bigcode/se_bcb_instruct_results", trust_remote_code=True, split="train") + + top_results = dict() + for sample in tqdm(retrieval_ds): + i, doc, score = sample["qid"], sample["tid"], sample["score"] + if score > 0.7: + if doc not in top_results: + top_results[doc] = (i, doc, score) + else: + if score > top_results[doc][2]: + top_results[doc] = (i, doc, score) + + top_id = {task_id: (qid, score) for qid, task_id, score in top_results.values()} + + lib_filter = {sample["task_id"] for sample in bcb if len(literal_eval(sample["libs"])) > 2} + length_filter = {sample["task_id"] for sample in bcb if len(sample["canonical_solution"]) > 426} + rate_filter = {task["task_id"]: task["solve_rate"] for task in solve_rate if task["solve_rate"] < 50} + + top_tid = top_id.keys() & length_filter & rate_filter.keys() & lib_filter + # hard_results = read_task_perf(top_tid) + + hard_bcb = bcb.filter(lambda x: x["task_id"] in top_tid) + hard_bcb_tid = hard_bcb["task_id"] + se_qid = [top_id[_id][0] for _id in hard_bcb_tid] + se_q = se.select(se_qid) + se_scores = [top_id[_id][1] for _id in hard_bcb_tid] + hard_bcb = hard_bcb.add_column("qid", se_qid) + hard_bcb = hard_bcb.add_column("question", se_q["question"]) + hard_bcb = hard_bcb.add_column("score", se_scores) + hard_bcb.push_to_hub("bigcode/bigcodebench-hard") \ No newline at end of file From c5268834e845e7fb77fe64a5128d08f650ff4dc7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 4 Jul 2024 06:23:39 +0800 Subject: [PATCH 033/325] print out update tokenizer --- analysis/get_results.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/analysis/get_results.py b/analysis/get_results.py index dfa9f56..742020b 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -17,6 +17,7 @@ def update_model_info(model_info): for model, info in model_info.items(): if "https://huggingface.co/" in info["link"]: hf_model = info["link"].split("https://huggingface.co/")[-1] + print(hf_model) tokenizer = AutoTokenizer.from_pretrained(hf_model, trust_remote_code=True) if tokenizer.chat_template is None: model_info[model]["direct_complete"] = True @@ -315,6 +316,16 @@ def push_ds(ds, path, local=False): files = [] complete_data, complete_files = read_task_perf("complete") instruct_data, instruct_files = read_task_perf("instruct") + + complete_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): + Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in complete_data.items()} + instruct_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): + Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in instruct_data.items()} + complete_ds = DatasetDict(complete_map) + instruct_ds = DatasetDict(instruct_map) + push_ds(complete_ds, "bigcode/bigcodebench-complete-perf") + push_ds(instruct_ds, "bigcode/bigcodebench-instruct-perf") + files.extend(complete_files) files.extend(instruct_files) shutil.rmtree("eval_results", ignore_errors=True) From 19ca466d67cf9881920bd69c7cb270c847cb4539 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 4 Jul 2024 15:51:34 +0800 Subject: [PATCH 034/325] fix: update the match cmd --- Docker/Evaluate.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index ca0a407..45b7758 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -33,4 +33,4 @@ USER bigcodebenchuser ENTRYPOINT ["python3", "-m", "bigcodebench.evaluate"] -CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep '^ *[0-9]\\+ bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/*"] \ No newline at end of file +CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/*"] \ No newline at end of file From b882aef255340e497b2f786807c7c47e65eddb15 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 5 Jul 2024 00:56:13 +0800 Subject: [PATCH 035/325] 89 models included --- analysis/utils.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index 65e8710..f954f6d 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -455,14 +455,14 @@ "open-data": "Partial", }, "microsoft/Phi-3-medium-128k-instruct": { - "name": "Phi-3-medium-128k-instruct", + "name": "Phi-3-Medium-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct", "prompted": True, "size": 14, "open-data": "None", }, "microsoft/Phi-3-small-128k-instruct": { - "name": "Phi-3-small-128k-instruct", + "name": "Phi-3-Small-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct", "prompted": True, "size": 7, @@ -594,4 +594,32 @@ "size": 7, "open-data": "Partial", }, + "new-microsoft/Phi-3-mini-128k-instruct": { + "name": "Phi-3-Mini-128K-Instruct (June 2024)", + "link": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct", + "prompted": True, + "size": 3.8, + "open-data": "None", + }, + "old-microsoft/Phi-3-mini-128k-instruct": { + "name": "Phi-3-Mini-128K-Instruct (Old)", + "link": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct", + "prompted": True, + "size": 3.8, + "open-data": "None", + }, + "internlm/internlm2_5-7b-chat": { + "name": "InternLM2.5-7B-Chat", + "link": "https://huggingface.co/internlm/internlm2_5-7b-chat", + "prompted": True, + "size": 7, + "open-data": "None", + }, + "NousResearch/Hermes-2-Pro-Llama-3-70B": { + "name": "Hermes-2-Pro-Llama-3-70B", + "link": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-70B", + "prompted": True, + "size": 70, + "open-data": "Partial", + }, } \ No newline at end of file From 6e53f6342f0ee3505b2f1a83b67c0366e95110c5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 5 Jul 2024 00:56:37 +0800 Subject: [PATCH 036/325] fix: update model names --- analysis/get_results.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index 742020b..4921931 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -49,11 +49,11 @@ def get_results(): for model, info in model_info.items(): model = model.replace("/", "--") hf_model = "" - if "https://huggingface.co/" in info["link"]: - hf_model = info["link"].split("https://huggingface.co/")[-1] - model = hf_model.replace("/", "--") files = glob(f"results/{model}--bigcodebench-*.json") assert files, f"No files found for results/{model}--bigcodebench-*.json" + # if "https://huggingface.co/" in info["link"]: + # hf_model = info["link"].split("https://huggingface.co/")[-1] + # model = hf_model.replace("/", "--") for file in files: _, suffix = os.path.basename(file).split("--bigcodebench-") status = [] @@ -152,8 +152,8 @@ def read_task_perf(task="complete"): task_perf = {f"BigCodeBench/{task_id}": 0 for task_id in range(1140)} model = model.replace("/", "--") - if info["link"].startswith("https://huggingface.co/"): - model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") + # if info["link"].startswith("https://huggingface.co/"): + # model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") try: if info["prompted"] and not info["direct_complete"]: files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") @@ -316,15 +316,15 @@ def push_ds(ds, path, local=False): files = [] complete_data, complete_files = read_task_perf("complete") instruct_data, instruct_files = read_task_perf("instruct") - - complete_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): - Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in complete_data.items()} - instruct_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): - Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in instruct_data.items()} - complete_ds = DatasetDict(complete_map) - instruct_ds = DatasetDict(instruct_map) - push_ds(complete_ds, "bigcode/bigcodebench-complete-perf") - push_ds(instruct_ds, "bigcode/bigcodebench-instruct-perf") + assert len(model_info) == len(complete_data) + # complete_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): + # Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in complete_data.items()} + # instruct_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): + # Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in instruct_data.items()} + # complete_ds = DatasetDict(complete_map) + # instruct_ds = DatasetDict(instruct_map) + # push_ds(complete_ds, "bigcode/bigcodebench-complete-perf") + # push_ds(instruct_ds, "bigcode/bigcodebench-instruct-perf") files.extend(complete_files) files.extend(instruct_files) From 694d73cef1cc14f3c161c19ea6ee0002809c62b2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 5 Jul 2024 23:42:49 +0800 Subject: [PATCH 037/325] feat: load hard data --- bigcodebench/data/bigcodebench.py | 16 +++++++++------- bigcodebench/data/utils.py | 14 +++++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index d7fea87..9b7a641 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -16,17 +16,19 @@ BIGCODEBENCH_HF = "bigcode/bigcodebench" BIGCODEBENCH_VERSION = "v0.1.0_hf" -def _ready_bigcodebench_path(mini=False, noextreme=False, version="default") -> str: +def _ready_bigcodebench_path(hard=False, version="default") -> str: if BIGCODEBENCH_OVERRIDE_PATH: return BIGCODEBENCH_OVERRIDE_PATH version = BIGCODEBENCH_VERSION if version == "default" else version url, path = get_dataset_metadata( - "BigCodeBench", BIGCODEBENCH_VERSION, mini, noextreme + BIGCODEBENCH_VERSION, hard ) + extra = "-hard" if hard else "" + try: - dataset = load_dataset(BIGCODEBENCH_HF, split=BIGCODEBENCH_VERSION) + dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) make_cache(url, dataset, path) except: if os.path.exists(path): @@ -37,7 +39,7 @@ def _ready_bigcodebench_path(mini=False, noextreme=False, version="default") -> def get_bigcodebench( - err_incomplete=True, mini=False, noextreme=False, version="default" + err_incomplete=True, hard=False, version="default" ) -> Dict[str, Dict]: """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts. @@ -54,19 +56,19 @@ def get_bigcodebench( """ # Check if open eval file exists in CACHE_DIR data_path = _ready_bigcodebench_path( - mini=mini, noextreme=noextreme, version=version + hard=hard, version=version ) data = {task["task_id"]: task for task in stream_jsonl(data_path)} if err_incomplete: completeness_check("BigCodeBench", data) return data -def get_bigcodebench_hash(mini=False, noextreme=False, version="default") -> str: +def get_bigcodebench_hash(hard=False, version="default") -> str: """Get the hash of BigCodeBench. Returns: str: The hash of BigCodeBench """ - data_path = _ready_bigcodebench_path(mini, noextreme, version="default") + data_path = _ready_bigcodebench_path(hard, version="default") with open(data_path, "rb") as f: data = f.read() return hashlib.md5(data).hexdigest() diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index 4d0a7ee..1921a0f 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -11,16 +11,12 @@ CACHE_DIR = user_cache_dir("bigcodebench") -def get_dataset_metadata(name: str, version: str, mini: bool, noextreme: bool = False): - assert name in ["BigCodeBench"], f"Unknown/unsupported dataset: {name}" +def get_dataset_metadata(version: str, hard: bool = False): extra = "" - assert not (mini and noextreme), "Cannot have both mini and noextreme" - if mini: - extra = "-Mini" - if noextreme: - extra = "-NoExtreme" - url = f"https://github.com/bigcode-project/bigcodebench-annotation/releases/download/{version}/{name}{extra}.jsonl.gz" - cache_path = os.path.join(CACHE_DIR, f"{name}{extra}-{version}.jsonl") + if hard: + extra = "-Hard" + url = f"https://github.com/bigcode-project/bigcodebench-annotation/releases/download/{version}/BigCodeBench{extra}.jsonl.gz" + cache_path = os.path.join(CACHE_DIR, f"BigCodeBench{extra}-{version}.jsonl") return url, cache_path From ce53dc142e9f0f505cce8359c1ed2e26243a4792 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 5 Jul 2024 23:43:37 +0800 Subject: [PATCH 038/325] feat: add support hard gen --- bigcodebench/generate.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 7edebfc..ea661ef 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -17,21 +17,23 @@ def codegen( model: DecoderBase, save_path: str, subset: str, + hard=False, greedy=False, strip_newlines=False, n_samples=1, id_range=None, resume=True, ): + extra = "Full" if not hard else "Hard" with Progress( - TextColumn(f"BigCodeBench--{subset} •" + "[progress.percentage]{task.percentage:>3.0f}%"), + TextColumn(f"BigCodeBench--{subset} ({extra}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), BarColumn(), MofNCompleteColumn(), TextColumn("•"), TimeElapsedColumn(), ) as p: - dataset = get_bigcodebench() + dataset = get_bigcodebench(hard=hard) if model.is_direct_completion() and subset == "instruct": raise Exception("Base model does not support direct completion for instruct tasks") @@ -106,6 +108,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", required=True, type=str) parser.add_argument("--subset", required=True, type=str) + parser.add_argument("--hard", action="store_true") parser.add_argument("--save_path", default=None, type=str) parser.add_argument("--bs", default=1, type=int) parser.add_argument("--n_samples", default=1, type=int) @@ -147,9 +150,10 @@ def main(): tp=args.tp, trust_remote_code=args.trust_remote_code ) - + + extra = "" if not args.hard else "-hard" if not args.save_path: - save_path = args.model.replace("/", "--") + f"--bigcodebench-{args.subset}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" + save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.subset}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" else: save_path = args.save_path @@ -157,6 +161,7 @@ def main(): model=model_runner, save_path=save_path, subset=args.subset, + hard=args.hard, greedy=args.greedy, strip_newlines=args.strip_newlines, n_samples=args.n_samples, From be21a9a182f809110b91cfcbd3bc42c3111f8535 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 5 Jul 2024 23:44:55 +0800 Subject: [PATCH 039/325] feat: add support hard eval --- bigcodebench/evaluate.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 6e94672..9764a0a 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -124,15 +124,15 @@ def evaluate(flags): assert flags.samples.endswith(".jsonl") result_path = flags.samples.replace(".jsonl", "_eval_results.json") - problems = get_bigcodebench() - dataset_hash = get_bigcodebench_hash() + problems = get_bigcodebench(hard=flags.hard) + dataset_hash = get_bigcodebench_hash(hard=flags.hard) if not flags.no_gt: expected_time = get_groundtruth(n_workers, problems, dataset_hash, flags.check_gt_only, flags.max_as_limit, flags.max_data_limit, flags.max_stack_limit) else: expected_time = {task_id: None for task_id in problems} - gt_pass_rate = np.mean([1 if v is not None else 0 for v in expected_time.values()]) + gt_pass_rate = np.mean([1 if v is not None else 0 for k, v in expected_time.items() if k in problems]) if os.path.isfile(result_path): print(f"Load from previous results from {result_path}") @@ -229,10 +229,12 @@ def stucking_checker(): ) # Calculate pass@k. - total = np.array([len(r) for r in results["eval"].values()]) + total = np.array([len(r) for k, r in results["eval"].items() if k in problems]) base_correct = [] - for res in results["eval"].values(): + for key, res in results["eval"].items(): + if key not in problems: + continue bc = sum([r["status"] == PASS for r in res]) base_correct.append(bc) @@ -245,8 +247,9 @@ def stucking_checker(): } mode = "-calibrated" if "sanitized-calibrated" in flags.samples else "" + extra = "Full" if not flags.hard else "Hard" flags.subset = flags.subset[0].upper() + flags.subset[1:] - cprint(f"BigCodeBench-{flags.subset}{mode}", "green") + cprint(f"BigCodeBench-{flags.subset}{mode} ({extra})", "green") if flags.no_gt: cprint(f"Groundtruth is not checked", "yellow") @@ -284,6 +287,7 @@ def main(): parser.add_argument( "--subset", required=True, type=str, choices=["complete", "instruct"] ) + parser.add_argument("--hard", action="store_true") parser.add_argument("--samples", required=True, type=str) parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) From fcaa7aa48b5b8206bc86e521b358d94ff82a0ccd Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 00:20:57 +0800 Subject: [PATCH 040/325] feat: rename subset --- bigcodebench/data/bigcodebench.py | 14 +++++++------- bigcodebench/data/utils.py | 6 ++---- bigcodebench/evaluate.py | 8 ++++---- bigcodebench/generate.py | 28 ++++++++++++++-------------- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 9b7a641..82ab515 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -16,16 +16,16 @@ BIGCODEBENCH_HF = "bigcode/bigcodebench" BIGCODEBENCH_VERSION = "v0.1.0_hf" -def _ready_bigcodebench_path(hard=False, version="default") -> str: +def _ready_bigcodebench_path(subset="", version="default") -> str: if BIGCODEBENCH_OVERRIDE_PATH: return BIGCODEBENCH_OVERRIDE_PATH version = BIGCODEBENCH_VERSION if version == "default" else version url, path = get_dataset_metadata( - BIGCODEBENCH_VERSION, hard + BIGCODEBENCH_VERSION, subset ) - extra = "-hard" if hard else "" + extra = "-subset" if subset else "" try: dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) @@ -39,7 +39,7 @@ def _ready_bigcodebench_path(hard=False, version="default") -> str: def get_bigcodebench( - err_incomplete=True, hard=False, version="default" + err_incomplete=True, subset="full", version="default" ) -> Dict[str, Dict]: """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts. @@ -56,19 +56,19 @@ def get_bigcodebench( """ # Check if open eval file exists in CACHE_DIR data_path = _ready_bigcodebench_path( - hard=hard, version=version + subset=subset, version=version ) data = {task["task_id"]: task for task in stream_jsonl(data_path)} if err_incomplete: completeness_check("BigCodeBench", data) return data -def get_bigcodebench_hash(hard=False, version="default") -> str: +def get_bigcodebench_hash(subset="", version="default") -> str: """Get the hash of BigCodeBench. Returns: str: The hash of BigCodeBench """ - data_path = _ready_bigcodebench_path(hard, version="default") + data_path = _ready_bigcodebench_path(subset, version="default") with open(data_path, "rb") as f: data = f.read() return hashlib.md5(data).hexdigest() diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index 1921a0f..9440d98 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -11,10 +11,8 @@ CACHE_DIR = user_cache_dir("bigcodebench") -def get_dataset_metadata(version: str, hard: bool = False): - extra = "" - if hard: - extra = "-Hard" +def get_dataset_metadata(version: str, subset: str=""): + extra = "-" + subset.capitalize() if subset else "" url = f"https://github.com/bigcode-project/bigcodebench-annotation/releases/download/{version}/BigCodeBench{extra}.jsonl.gz" cache_path = os.path.join(CACHE_DIR, f"BigCodeBench{extra}-{version}.jsonl") return url, cache_path diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 9764a0a..ec36097 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -247,9 +247,9 @@ def stucking_checker(): } mode = "-calibrated" if "sanitized-calibrated" in flags.samples else "" - extra = "Full" if not flags.hard else "Hard" - flags.subset = flags.subset[0].upper() + flags.subset[1:] - cprint(f"BigCodeBench-{flags.subset}{mode} ({extra})", "green") + extra = flags.subset.capitalize() + flags.split = flags.split.capitalize() + cprint(f"BigCodeBench-{flags.split}{mode} ({extra})", "green") if flags.no_gt: cprint(f"Groundtruth is not checked", "yellow") @@ -285,7 +285,7 @@ def stucking_checker(): def main(): parser = argparse.ArgumentParser() parser.add_argument( - "--subset", required=True, type=str, choices=["complete", "instruct"] + "--split", required=True, type=str, choices=["complete", "instruct"] ) parser.add_argument("--hard", action="store_true") parser.add_argument("--samples", required=True, type=str) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index ea661ef..c6e08ed 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -16,26 +16,26 @@ def codegen( model: DecoderBase, save_path: str, - subset: str, - hard=False, + split: str, + subset="full", greedy=False, strip_newlines=False, n_samples=1, id_range=None, resume=True, ): - extra = "Full" if not hard else "Hard" + extra = "-" + subset.capitalize() if subset else "" with Progress( - TextColumn(f"BigCodeBench--{subset} ({extra}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), + TextColumn(f"BigCodeBench--{split} ({extra}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), BarColumn(), MofNCompleteColumn(), TextColumn("•"), TimeElapsedColumn(), ) as p: - dataset = get_bigcodebench(hard=hard) + dataset = get_bigcodebench(subset=subset) - if model.is_direct_completion() and subset == "instruct": + if model.is_direct_completion() and split == "instruct": raise Exception("Base model does not support direct completion for instruct tasks") # create save_path if it doesn't exist, e.g., a/b.jsonl @@ -72,9 +72,9 @@ def codegen( sidx = n_samples - nsamples while sidx < n_samples: try: - prompt = task[f"{subset}_prompt"] + prompt = task[f"{split}_prompt"] except: - raise Exception(f"Invalid subset {subset}") + raise Exception(f"Invalid split {split}") if strip_newlines: prompt = prompt.strip("\n") outputs = model.codegen( @@ -107,8 +107,8 @@ def codegen( def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", required=True, type=str) - parser.add_argument("--subset", required=True, type=str) - parser.add_argument("--hard", action="store_true") + parser.add_argument("--split", required=True, type=str) + parser.add_argument("--subset", default="", type=str) parser.add_argument("--save_path", default=None, type=str) parser.add_argument("--bs", default=1, type=int) parser.add_argument("--n_samples", default=1, type=int) @@ -124,7 +124,7 @@ def main(): args = parser.parse_args() - assert args.subset in ["complete", "instruct"], f"Invalid subset {args.subset}" + assert args.split in ["complete", "instruct"], f"Invalid split {args.split}" assert args.backend in ["vllm", "hf", "openai", "mistral", "anthropic", "google"] if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\ @@ -151,17 +151,17 @@ def main(): trust_remote_code=args.trust_remote_code ) - extra = "" if not args.hard else "-hard" + extra = "-"+args.subset if args.subset if not args.save_path: - save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.subset}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" + save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" else: save_path = args.save_path codegen( model=model_runner, save_path=save_path, + split=args.split, subset=args.subset, - hard=args.hard, greedy=args.greedy, strip_newlines=args.strip_newlines, n_samples=args.n_samples, From f4f62b46108b5b52483e2e93612690d9c8bb0b46 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 00:30:48 +0800 Subject: [PATCH 041/325] fix: change subset namings --- bigcodebench/data/bigcodebench.py | 4 ++-- bigcodebench/evaluate.py | 6 +++--- bigcodebench/generate.py | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 82ab515..6a48bc2 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -25,7 +25,7 @@ def _ready_bigcodebench_path(subset="", version="default") -> str: BIGCODEBENCH_VERSION, subset ) - extra = "-subset" if subset else "" + extra = "-" + subset if subset else "" try: dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) @@ -39,7 +39,7 @@ def _ready_bigcodebench_path(subset="", version="default") -> str: def get_bigcodebench( - err_incomplete=True, subset="full", version="default" + err_incomplete=True, subset="", version="default" ) -> Dict[str, Dict]: """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts. diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index ec36097..f74623a 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -124,8 +124,8 @@ def evaluate(flags): assert flags.samples.endswith(".jsonl") result_path = flags.samples.replace(".jsonl", "_eval_results.json") - problems = get_bigcodebench(hard=flags.hard) - dataset_hash = get_bigcodebench_hash(hard=flags.hard) + problems = get_bigcodebench(subset=flags.subset) + dataset_hash = get_bigcodebench_hash(subset=flags.subset) if not flags.no_gt: expected_time = get_groundtruth(n_workers, problems, dataset_hash, flags.check_gt_only, flags.max_as_limit, flags.max_data_limit, flags.max_stack_limit) @@ -287,7 +287,7 @@ def main(): parser.add_argument( "--split", required=True, type=str, choices=["complete", "instruct"] ) - parser.add_argument("--hard", action="store_true") + parser.add_argument("--subset", default="", choices=["", "hard"], type=str) parser.add_argument("--samples", required=True, type=str) parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index c6e08ed..3335f84 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -125,6 +125,7 @@ def main(): assert args.split in ["complete", "instruct"], f"Invalid split {args.split}" + assert args.subset in ["", "hard"], f"Invalid subset {args.subset}" assert args.backend in ["vllm", "hf", "openai", "mistral", "anthropic", "google"] if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\ From ba1b36bd5d21bf410a074193e5af8ea31187c8a1 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 00:38:06 +0800 Subject: [PATCH 042/325] fix: change default subset names --- bigcodebench/data/bigcodebench.py | 6 +++--- bigcodebench/data/utils.py | 4 ++-- bigcodebench/evaluate.py | 2 +- bigcodebench/generate.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 6a48bc2..30a13e8 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -16,7 +16,7 @@ BIGCODEBENCH_HF = "bigcode/bigcodebench" BIGCODEBENCH_VERSION = "v0.1.0_hf" -def _ready_bigcodebench_path(subset="", version="default") -> str: +def _ready_bigcodebench_path(subset="full", version="default") -> str: if BIGCODEBENCH_OVERRIDE_PATH: return BIGCODEBENCH_OVERRIDE_PATH @@ -39,7 +39,7 @@ def _ready_bigcodebench_path(subset="", version="default") -> str: def get_bigcodebench( - err_incomplete=True, subset="", version="default" + err_incomplete=True, subset="full", version="default" ) -> Dict[str, Dict]: """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts. @@ -63,7 +63,7 @@ def get_bigcodebench( completeness_check("BigCodeBench", data) return data -def get_bigcodebench_hash(subset="", version="default") -> str: +def get_bigcodebench_hash(subset="full", version="default") -> str: """Get the hash of BigCodeBench. Returns: str: The hash of BigCodeBench diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index 9440d98..c9fe7e8 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -11,8 +11,8 @@ CACHE_DIR = user_cache_dir("bigcodebench") -def get_dataset_metadata(version: str, subset: str=""): - extra = "-" + subset.capitalize() if subset else "" +def get_dataset_metadata(version: str, subset: str="full"): + extra = "-" + subset.capitalize() if subset != "full" else "" url = f"https://github.com/bigcode-project/bigcodebench-annotation/releases/download/{version}/BigCodeBench{extra}.jsonl.gz" cache_path = os.path.join(CACHE_DIR, f"BigCodeBench{extra}-{version}.jsonl") return url, cache_path diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index f74623a..015b93b 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -287,7 +287,7 @@ def main(): parser.add_argument( "--split", required=True, type=str, choices=["complete", "instruct"] ) - parser.add_argument("--subset", default="", choices=["", "hard"], type=str) + parser.add_argument("--subset", default="full", choices=["full", "hard"], type=str) parser.add_argument("--samples", required=True, type=str) parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 3335f84..4411601 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -24,7 +24,7 @@ def codegen( id_range=None, resume=True, ): - extra = "-" + subset.capitalize() if subset else "" + extra = "-" + subset.capitalize() with Progress( TextColumn(f"BigCodeBench--{split} ({extra}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), BarColumn(), @@ -108,7 +108,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", required=True, type=str) parser.add_argument("--split", required=True, type=str) - parser.add_argument("--subset", default="", type=str) + parser.add_argument("--subset", default="full", type=str) parser.add_argument("--save_path", default=None, type=str) parser.add_argument("--bs", default=1, type=int) parser.add_argument("--n_samples", default=1, type=int) @@ -125,7 +125,7 @@ def main(): assert args.split in ["complete", "instruct"], f"Invalid split {args.split}" - assert args.subset in ["", "hard"], f"Invalid subset {args.subset}" + assert args.subset in ["full", "hard"], f"Invalid subset {args.subset}" assert args.backend in ["vllm", "hf", "openai", "mistral", "anthropic", "google"] if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\ From 38ba18cdce4a080edc39ee67f98c3b3948a0f9d0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 00:41:03 +0800 Subject: [PATCH 043/325] fix: update choices --- bigcodebench/evaluate.py | 2 +- bigcodebench/generate.py | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 015b93b..287f174 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -287,7 +287,7 @@ def main(): parser.add_argument( "--split", required=True, type=str, choices=["complete", "instruct"] ) - parser.add_argument("--subset", default="full", choices=["full", "hard"], type=str) + parser.add_argument("--subset", default="full", type=str, choices=["full", "hard"]) parser.add_argument("--samples", required=True, type=str) parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 4411601..ee8f23b 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -107,8 +107,8 @@ def codegen( def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", required=True, type=str) - parser.add_argument("--split", required=True, type=str) - parser.add_argument("--subset", default="full", type=str) + parser.add_argument("--split", required=True, type=str, choices=["complete", "instruct"]) + parser.add_argument("--subset", default="full", type=str, choices=["full", "hard"]) parser.add_argument("--save_path", default=None, type=str) parser.add_argument("--bs", default=1, type=int) parser.add_argument("--n_samples", default=1, type=int) @@ -117,17 +117,12 @@ def main(): parser.add_argument("--strip_newlines", action="store_true") parser.add_argument("--resume", action="store_true") parser.add_argument("--id_range", nargs=2, type=int) - parser.add_argument("--backend", default="vllm", type=str) + parser.add_argument("--backend", default="vllm", type=str, choices=["vllm", "hf", "openai", "mistral", "anthropic", "google"]) parser.add_argument("--base_url", default=None, type=str) parser.add_argument("--tp", default=1, type=int) parser.add_argument("--trust_remote_code", action="store_true") args = parser.parse_args() - - assert args.split in ["complete", "instruct"], f"Invalid split {args.split}" - assert args.subset in ["full", "hard"], f"Invalid subset {args.subset}" - assert args.backend in ["vllm", "hf", "openai", "mistral", "anthropic", "google"] - if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\ or (args.temperature == 0 and args.n_samples == 1): args.temperature = 0 From ce634831259626ba2ad27dfdbad1b4bea58614ec Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 01:46:45 +0800 Subject: [PATCH 044/325] fix: init new hard subset --- analysis/bcb_hard.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/analysis/bcb_hard.py b/analysis/bcb_hard.py index e12c3b1..facc05e 100644 --- a/analysis/bcb_hard.py +++ b/analysis/bcb_hard.py @@ -6,7 +6,7 @@ from glob import glob from sentence_transformers import SentenceTransformer, util import matplotlib.pyplot as plt -from datasets import load_dataset, Dataset, Features, Value, Sequence +from datasets import load_dataset, Dataset, Features, Value, Sequence, DatasetDict from utils import * @@ -117,12 +117,27 @@ def read_task_perf(top_tid, task="complete"): top_tid = top_id.keys() & length_filter & rate_filter.keys() & lib_filter # hard_results = read_task_perf(top_tid) - hard_bcb = bcb.filter(lambda x: x["task_id"] in top_tid) + filtered_bcb = bcb.filter(lambda x: x["task_id"] in top_tid) hard_bcb_tid = hard_bcb["task_id"] se_qid = [top_id[_id][0] for _id in hard_bcb_tid] se_q = se.select(se_qid) se_scores = [top_id[_id][1] for _id in hard_bcb_tid] - hard_bcb = hard_bcb.add_column("qid", se_qid) - hard_bcb = hard_bcb.add_column("question", se_q["question"]) - hard_bcb = hard_bcb.add_column("score", se_scores) - hard_bcb.push_to_hub("bigcode/bigcodebench-hard") \ No newline at end of file + + hard_bcb_dict = { + "task_id": [f"BigCodeBenchHard/{i}" for i in range(len(hard_bcb))], + "complete_prompt": hard_bcb["complete_prompt"], + "instruct_prompt": hard_bcb["instruct_prompt"], + "canonical_solution": hard_bcb["canonical_solution"], + "code_prompt": hard_bcb["code_prompt"], + "test": hard_bcb["test"], + "entry_point": hard_bcb["entry_point"], + "doc_struct": hard_bcb["doc_struct"], + "libs": hard_bcb["libs"], + "q_idx": se_qid, + "question": se_q["question"], + "score": se_scores, + "_id": hard_bcb_tid + } + + hard_bcb = Dataset.from_dict(hard_bcb_dict) + DatasetDict({"v0.1.0_hf": hard_bcb}).push_to_hub("bigcode/bigcodebench-hard") \ No newline at end of file From 9099dfa196cd9e0d9edffaba6ea8e82c834288fb Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 02:08:56 +0800 Subject: [PATCH 045/325] fix: update subset --- bigcodebench/generate.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index ee8f23b..24f21cb 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -24,9 +24,8 @@ def codegen( id_range=None, resume=True, ): - extra = "-" + subset.capitalize() with Progress( - TextColumn(f"BigCodeBench--{split} ({extra}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), + TextColumn(f"BigCodeBench--{split} ({subset.capitalize()}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), BarColumn(), MofNCompleteColumn(), TextColumn("•"), @@ -147,9 +146,9 @@ def main(): trust_remote_code=args.trust_remote_code ) - extra = "-"+args.subset if args.subset + extra = "-" + args.subset if args.subset else "" if not args.save_path: - save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" + save_path = args.model.replace("/", "--") + f"--bigcodebench-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" else: save_path = args.save_path From 52a9382158f914e1ae87e1354020b12ed744f964 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 02:09:35 +0800 Subject: [PATCH 046/325] fix: capitalize split --- bigcodebench/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 24f21cb..b0c3b39 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -25,7 +25,7 @@ def codegen( resume=True, ): with Progress( - TextColumn(f"BigCodeBench--{split} ({subset.capitalize()}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), + TextColumn(f"BigCodeBench--{split.capitalize()} ({subset.capitalize()}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), BarColumn(), MofNCompleteColumn(), TextColumn("•"), From 69d2e424b927dacb2ebf57e363cfed1e73f71842 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 02:11:30 +0800 Subject: [PATCH 047/325] fix: add extra name in save ppath --- bigcodebench/data/bigcodebench.py | 14 +++++++------- bigcodebench/data/utils.py | 9 ++++----- bigcodebench/generate.py | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 30a13e8..38f66cf 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -27,13 +27,13 @@ def _ready_bigcodebench_path(subset="full", version="default") -> str: extra = "-" + subset if subset else "" - try: - dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) - make_cache(url, dataset, path) - except: - if os.path.exists(path): - os.remove(path) - make_cache(url, None, path, gh=True) + # try: + dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) + make_cache(url, dataset, path) + # except: + # if os.path.exists(path): + # os.remove(path) + # make_cache(url, None, path, gh=True) return path diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index c9fe7e8..858897b 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -20,9 +20,8 @@ def get_dataset_metadata(version: str, subset: str="full"): def make_cache(gzip_url, hf_data, cache_path, gh=False): # Check if open eval file exists in CACHE_DIR - if not os.path.exists(cache_path): - - if gh: + if gh: + if not os.path.exists(cache_path): # Install BigCodeBench dataset and parse as jsonl print(f"Downloading dataset from {gzip_url}") with tempdir.TempDir() as tmpdir: @@ -39,8 +38,8 @@ def make_cache(gzip_url, hf_data, cache_path, gh=False): # Write the original open eval file to CACHE_DIR with open(cache_path, "w") as f: f.write(data) - else: - hf_data.to_json(cache_path) + else: + hf_data.to_json(cache_path) def write_jsonl( diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index b0c3b39..f6580be 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -148,7 +148,7 @@ def main(): extra = "-" + args.subset if args.subset else "" if not args.save_path: - save_path = args.model.replace("/", "--") + f"--bigcodebench-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" + save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" else: save_path = args.save_path From 259132248d53b44d683aac0b880e539b8b2d8440 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 02:11:55 +0800 Subject: [PATCH 048/325] uncomment --- bigcodebench/data/bigcodebench.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 38f66cf..30a13e8 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -27,13 +27,13 @@ def _ready_bigcodebench_path(subset="full", version="default") -> str: extra = "-" + subset if subset else "" - # try: - dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) - make_cache(url, dataset, path) - # except: - # if os.path.exists(path): - # os.remove(path) - # make_cache(url, None, path, gh=True) + try: + dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) + make_cache(url, dataset, path) + except: + if os.path.exists(path): + os.remove(path) + make_cache(url, None, path, gh=True) return path From 9c8ac7a1499923845233a9d28db3cdfb5d194c54 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 03:42:43 +0800 Subject: [PATCH 049/325] fix: change task id to idx --- bigcodebench/generate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index f6580be..b809a19 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -41,9 +41,8 @@ def codegen( dirname = os.path.dirname(save_path) if not os.path.exists(dirname) and dirname != "": os.makedirs(dirname) - for task_id, task in p.track(dataset.items()): + for id_num, (task_id, task) in enumerate(p.track(dataset.items())): if id_range is not None: - id_num = int(task_id.split("/")[1]) low, high = id_range if id_num < low or id_num >= high: p.console.print(f"Skipping {task_id} as it is not in {id_range}") From 3488a355ad096ea2ab4ac3ec282c4f10af4e52c0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 03:43:14 +0800 Subject: [PATCH 050/325] update bcb hard format --- analysis/bcb_hard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysis/bcb_hard.py b/analysis/bcb_hard.py index facc05e..28b54f6 100644 --- a/analysis/bcb_hard.py +++ b/analysis/bcb_hard.py @@ -124,7 +124,7 @@ def read_task_perf(top_tid, task="complete"): se_scores = [top_id[_id][1] for _id in hard_bcb_tid] hard_bcb_dict = { - "task_id": [f"BigCodeBenchHard/{i}" for i in range(len(hard_bcb))], + "task_id": hard_bcb_tid, "complete_prompt": hard_bcb["complete_prompt"], "instruct_prompt": hard_bcb["instruct_prompt"], "canonical_solution": hard_bcb["canonical_solution"], From ed30e0a5a599dff1359ca3ecc9ed2ddd69bbb496 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 03:50:22 +0800 Subject: [PATCH 051/325] fix: update kill cmd --- Docker/Evaluate.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index ca0a407..45b7758 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -33,4 +33,4 @@ USER bigcodebenchuser ENTRYPOINT ["python3", "-m", "bigcodebench.evaluate"] -CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep '^ *[0-9]\\+ bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/*"] \ No newline at end of file +CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/*"] \ No newline at end of file From 0695a63cfe3267ba4b33dd79f63a353b119f7c2e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 04:48:50 +0800 Subject: [PATCH 052/325] fix: adjust time limit --- bigcodebench/eval/__init__.py | 3 ++- bigcodebench/eval/utils.py | 2 ++ bigcodebench/evaluate.py | 4 ++-- bigcodebench/gen/util/__init__.py | 7 ++++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bigcodebench/eval/__init__.py b/bigcodebench/eval/__init__.py index 2c2cdfc..d3f6304 100644 --- a/bigcodebench/eval/__init__.py +++ b/bigcodebench/eval/__init__.py @@ -41,6 +41,7 @@ swallow_io, time_limit, safe_environment, + TIMEOUT, ) @@ -178,7 +179,7 @@ def untrusted_check( gt_time_limit: float = 60 ) -> Tuple[str, np.ndarray]: time_limit = max(min_time_limit, gt_time_limit) - timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", 120), time_limit) + 1 + timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT), time_limit) + 1 # shared memory objects stat = Value("i", _UNKNOWN) manager = Manager() diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index a7ff216..bf1861e 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -31,6 +31,8 @@ import multiprocessing from typing import Optional +TIMEOUT=240 + @contextlib.contextmanager def swallow_subprocess_output(): """Context manager to swallow stdout and stderr for subprocesses.""" diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 287f174..2a6a573 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -199,11 +199,11 @@ def evaluate(flags): def stucking_checker(): while remainings: last_size = len(remainings) - time.sleep(120) + time.sleep(240) if last_size != len(remainings) or len(remainings) == 0: continue # Potential stucking - warn("No samples had finished testing in the last 120s") + warn("No samples had finished testing in the last 240s") warn(f"{len(remainings)} samples to be tested: {remainings}") threading.Thread(target=stucking_checker).start() diff --git a/bigcodebench/gen/util/__init__.py b/bigcodebench/gen/util/__init__.py index 6de3468..78f903a 100644 --- a/bigcodebench/gen/util/__init__.py +++ b/bigcodebench/gen/util/__init__.py @@ -11,6 +11,7 @@ swallow_io, time_limit, safe_environment, + TIMEOUT, ) @@ -51,7 +52,7 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta suite = loader.loadTestsFromTestCase(TestCases) test_result = unittest.TestResult() start = time.time() - with safe_environment(), swallow_io(), time_limit(seconds=120): + with safe_environment(), swallow_io(), time_limit(seconds=TIMEOUT): suite.run(test_result) if len(test_result.failures + test_result.errors) > 0: @@ -68,7 +69,7 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta def trusted_check_exec(code, inputs): """Check trusted_exec success.""" try: - with time_limit(seconds=120): + with time_limit(seconds=TIMEOUT): trusted_exec(code, inputs) except Exception: return False @@ -83,7 +84,7 @@ def trusted_check( max_data_limit: float, max_stack_limit: float, ): - timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", 120) + 1 + timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT) + 1 # shared memory objects times = Value("d", -1) manager = Manager() From 7b688cd7490ddf0a1031b915d864523740e021be Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 04:58:59 +0800 Subject: [PATCH 053/325] fix: change result path --- bigcodebench/evaluate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 2a6a573..59ad079 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -118,11 +118,12 @@ def evaluate(flags): # bypass the samples flags.samples = "__dummy__.jsonl" + extra = flags.subset + "_" if flags.subset != "full" else "" if os.path.isdir(flags.samples): - result_path = os.path.join(flags.samples, "eval_results.json") + result_path = os.path.join(flags.samples, f"{extra}eval_results.json") else: assert flags.samples.endswith(".jsonl") - result_path = flags.samples.replace(".jsonl", "_eval_results.json") + result_path = flags.samples.replace(".jsonl", f"_{extra}eval_results.json") problems = get_bigcodebench(subset=flags.subset) dataset_hash = get_bigcodebench_hash(subset=flags.subset) From 4a23976c8d0615460e41f94922ff9f3a76b44e0a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 21:18:30 +0800 Subject: [PATCH 054/325] fix: update generate save path --- bigcodebench/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index b809a19..6f9c35e 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -145,7 +145,7 @@ def main(): trust_remote_code=args.trust_remote_code ) - extra = "-" + args.subset if args.subset else "" + extra = "-" + args.subset if args.subset != "full" else "" if not args.save_path: save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" else: From fdee310d15343d352da1d0292ffbb0b7e06b7646 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 6 Jul 2024 21:46:22 +0800 Subject: [PATCH 055/325] fix: update download name --- bigcodebench/data/bigcodebench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 30a13e8..b113944 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -25,7 +25,7 @@ def _ready_bigcodebench_path(subset="full", version="default") -> str: BIGCODEBENCH_VERSION, subset ) - extra = "-" + subset if subset else "" + extra = "-" + subset if subset != "full" else "" try: dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) From 3e543ec7e79b9b507324fdc5490c76b275477408 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 7 Jul 2024 20:18:11 +0800 Subject: [PATCH 056/325] fix: timeout type --- bigcodebench/eval/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index bf1861e..8b77718 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -31,7 +31,7 @@ import multiprocessing from typing import Optional -TIMEOUT=240 +TIMEOUT=240.0 @contextlib.contextmanager def swallow_subprocess_output(): From c355738484ff6505531acfc92760dd744524e1d5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 7 Jul 2024 21:16:34 +0800 Subject: [PATCH 057/325] fix: resolve constant conflict --- bigcodebench/eval/__init__.py | 4 ++-- bigcodebench/eval/utils.py | 2 +- bigcodebench/gen/util/__init__.py | 8 ++++---- bigcodebench/gen/util/openai_request.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bigcodebench/eval/__init__.py b/bigcodebench/eval/__init__.py index d3f6304..1a8fcfc 100644 --- a/bigcodebench/eval/__init__.py +++ b/bigcodebench/eval/__init__.py @@ -41,7 +41,7 @@ swallow_io, time_limit, safe_environment, - TIMEOUT, + TIMEOUT_LIMIT, ) @@ -179,7 +179,7 @@ def untrusted_check( gt_time_limit: float = 60 ) -> Tuple[str, np.ndarray]: time_limit = max(min_time_limit, gt_time_limit) - timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT), time_limit) + 1 + timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT), time_limit) + 1 # shared memory objects stat = Value("i", _UNKNOWN) manager = Manager() diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index 8b77718..844d2ea 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -31,7 +31,7 @@ import multiprocessing from typing import Optional -TIMEOUT=240.0 +TIMEOUT_LIMIT=240.0 @contextlib.contextmanager def swallow_subprocess_output(): diff --git a/bigcodebench/gen/util/__init__.py b/bigcodebench/gen/util/__init__.py index 78f903a..f8f6238 100644 --- a/bigcodebench/gen/util/__init__.py +++ b/bigcodebench/gen/util/__init__.py @@ -11,7 +11,7 @@ swallow_io, time_limit, safe_environment, - TIMEOUT, + TIMEOUT_LIMIT, ) @@ -52,7 +52,7 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta suite = loader.loadTestsFromTestCase(TestCases) test_result = unittest.TestResult() start = time.time() - with safe_environment(), swallow_io(), time_limit(seconds=TIMEOUT): + with safe_environment(), swallow_io(), time_limit(seconds=TIMEOUT_LIMIT): suite.run(test_result) if len(test_result.failures + test_result.errors) > 0: @@ -69,7 +69,7 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta def trusted_check_exec(code, inputs): """Check trusted_exec success.""" try: - with time_limit(seconds=TIMEOUT): + with time_limit(seconds=TIMEOUT_LIMIT): trusted_exec(code, inputs) except Exception: return False @@ -84,7 +84,7 @@ def trusted_check( max_data_limit: float, max_stack_limit: float, ): - timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT) + 1 + timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT) + 1 # shared memory objects times = Value("d", -1) manager = Manager() diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index e347ffe..2254f5a 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -29,8 +29,8 @@ def make_request( ], max_tokens=max_tokens, temperature=temperature, - n=n, - **kwargs + # n=n, + # **kwargs ) From 02b396a9fe8af61e750b084041694cf4ed46d8bc Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 7 Jul 2024 21:18:47 +0800 Subject: [PATCH 058/325] fix: undoo commented lines --- bigcodebench/gen/util/openai_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index 2254f5a..e347ffe 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -29,8 +29,8 @@ def make_request( ], max_tokens=max_tokens, temperature=temperature, - # n=n, - # **kwargs + n=n, + **kwargs ) From fc759a233d123f36587bbdadad6509babf000a75 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 8 Jul 2024 05:16:01 +0800 Subject: [PATCH 059/325] fix: avoid duplicated save --- bigcodebench/data/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index 858897b..fa91abe 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -20,8 +20,9 @@ def get_dataset_metadata(version: str, subset: str="full"): def make_cache(gzip_url, hf_data, cache_path, gh=False): # Check if open eval file exists in CACHE_DIR - if gh: - if not os.path.exists(cache_path): + + if not os.path.exists(cache_path): + if gh: # Install BigCodeBench dataset and parse as jsonl print(f"Downloading dataset from {gzip_url}") with tempdir.TempDir() as tmpdir: @@ -38,8 +39,8 @@ def make_cache(gzip_url, hf_data, cache_path, gh=False): # Write the original open eval file to CACHE_DIR with open(cache_path, "w") as f: f.write(data) - else: - hf_data.to_json(cache_path) + else: + hf_data.to_json(cache_path) def write_jsonl( From 32324d2de6c942580f98436c8595c0b6f16ae229 Mon Sep 17 00:00:00 2001 From: marianna13 Date: Mon, 8 Jul 2024 15:28:45 +0200 Subject: [PATCH 060/325] save pass@k to json file --- bigcodebench/evaluate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index b761530..446c533 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -276,6 +276,12 @@ def stucking_checker(): if not os.path.isfile(result_path): with open(result_path, "w") as f: json.dump(results, f, indent=2) + + pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") + pass_at_k["model"] = flags.samples.split("/")[-1].replace(".jsonl", "") + pass_at_k["subset"] = flags.subset + with open(pass_at_k_path, "w") as f: + json.dump(pass_at_k, f, indent=2) def main(): From b2a14b636c6ae72dd0ebd8574d63cdd1fea80613 Mon Sep 17 00:00:00 2001 From: marianna13 Date: Mon, 8 Jul 2024 15:29:34 +0200 Subject: [PATCH 061/325] add tokenizer_name argument for custom tokenizer --- bigcodebench/generate.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 7edebfc..ca21560 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -35,7 +35,7 @@ def codegen( if model.is_direct_completion() and subset == "instruct": raise Exception("Base model does not support direct completion for instruct tasks") - + # create save_path if it doesn't exist, e.g., a/b.jsonl dirname = os.path.dirname(save_path) if not os.path.exists(dirname) and dirname != "": @@ -118,6 +118,8 @@ def main(): parser.add_argument("--base_url", default=None, type=str) parser.add_argument("--tp", default=1, type=int) parser.add_argument("--trust_remote_code", action="store_true") + parser.add_argument("--tokenizer_name", default=None, type=str) + args = parser.parse_args() @@ -145,7 +147,8 @@ def main(): temperature=args.temperature, base_url=args.base_url, tp=args.tp, - trust_remote_code=args.trust_remote_code + trust_remote_code=args.trust_remote_code, + tokenizer_name=args.tokenizer_name ) if not args.save_path: @@ -161,7 +164,7 @@ def main(): strip_newlines=args.strip_newlines, n_samples=args.n_samples, resume=args.resume, - id_range=args.id_range, + id_range=args.id_range ) From 96aafc0a0737590f763ec49337ab4c305dfe3fb4 Mon Sep 17 00:00:00 2001 From: marianna13 Date: Mon, 8 Jul 2024 15:30:02 +0200 Subject: [PATCH 062/325] add custom tokenzier --- bigcodebench/model.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/bigcodebench/model.py b/bigcodebench/model.py index 19110d5..b2b89fa 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -92,6 +92,7 @@ def __init__( max_new_tokens: int = 1280, dtype: str = "bfloat16", # default trust_remote_code: bool = False, + tokenizer_name: str = None, ) -> None: print("Initializing a decoder model: {} ...".format(name)) self.name = name @@ -102,6 +103,7 @@ def __init__( self.max_new_tokens = max_new_tokens self.dtype = dtype self.trust_remote_code = trust_remote_code + self.tokenizer_name = tokenizer_name @abstractmethod def codegen( @@ -129,11 +131,13 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: "dtype": self.dtype, "trust_remote_code": self.trust_remote_code, } - - self.tokenizer = AutoTokenizer.from_pretrained(self.name, **kwargs) + if self.tokenizer_name is None: + self.tokenizer_name = self.name + + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) - self.llm = LLM(model=name, max_model_len=2048, **kwargs) + self.llm = LLM(model=name, max_model_len=2048, tokenizer=self.tokenizer_name, **kwargs) def is_direct_completion(self) -> bool: return self.tokenizer.chat_template is None @@ -185,9 +189,12 @@ def __init__(self, name: str, dataset: str, **kwargs): kwargs["torch_dtype"] = getattr(torch, self.dtype) self.skip_special_tokens = True - print(f"{kwargs = }") + print(f"{kwargs = }", self.tokenizer_name) + + if self.tokenizer_name is None: + self.tokenizer_name = self.name - self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) @@ -253,7 +260,7 @@ def __init__(self, name: str, **kwargs): super().__init__(name=name, **kwargs) self.eos += ["\n```\n"] print(f"EOS strings: {self.eos}") - self.tokenizer = AutoTokenizer.from_pretrained(self.name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name, **kwargs) def codegen( self, prompt: str, do_sample: bool = True, num_samples: int = 200 @@ -486,6 +493,7 @@ def make_model( tp=1, base_url=None, trust_remote_code=False, + tokenizer_name=None, ): if backend == "vllm": return GeneralVllmDecoder( @@ -495,6 +503,7 @@ def make_model( dataset=dataset, tp=tp, trust_remote_code=trust_remote_code, + tokenizer_name=tokenizer_name, ) elif backend == "hf": return GenenralHfTorchDecoder( @@ -503,6 +512,7 @@ def make_model( temperature=temperature, dataset=dataset, trust_remote_code=trust_remote_code, + tokenizer_name=tokenizer_name, ) elif backend == "openai": return OpenAIChatDecoder( From 7d9e4fc53978f1be2fe6fccee770a21d9b05bef5 Mon Sep 17 00:00:00 2001 From: marianna13 Date: Tue, 9 Jul 2024 18:37:50 +0200 Subject: [PATCH 063/325] ask user whther to save pass@k --- bigcodebench/evaluate.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index a3495ea..6805253 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -281,8 +281,28 @@ def stucking_checker(): pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") pass_at_k["model"] = flags.samples.split("/")[-1].replace(".jsonl", "") pass_at_k["subset"] = flags.subset - with open(pass_at_k_path, "w") as f: - json.dump(pass_at_k, f, indent=2) + + def save_pass_at_k(): + with open(pass_at_k_path, "w") as f: + json.dump(pass_at_k, f, indent=2) + + if os.path.isfile(pass_at_k_path): + saved_pass_at_k = json.load(open(pass_at_k_path, "r")) + # compare saved_pass_at_k with pass_at_k + for k in saved_pass_at_k.keys(): + if pass_at_k[k] != saved_pass_at_k[k]: + cprint(f"Warning: {k} is different from the saved one", "yellow") + + # ask user whether to save the pass@k + decision = "" + while decision.lower() not in ["y", "n"]: + print(f"Save pass@k to {pass_at_k_path}? [Y/N]") + decision = input() + if decision.lower() == "y": + save_pass_at_k() + + else: + save_pass_at_k() def main(): From 170c9a01840aa1e4ee37d9aa52d5fc7f1a4bcc1f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 15 Jul 2024 06:54:07 +0800 Subject: [PATCH 064/325] add more models --- analysis/bcb_hard.py | 143 ------------------------ analysis/utils.py | 261 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 251 insertions(+), 153 deletions(-) delete mode 100644 analysis/bcb_hard.py diff --git a/analysis/bcb_hard.py b/analysis/bcb_hard.py deleted file mode 100644 index 28b54f6..0000000 --- a/analysis/bcb_hard.py +++ /dev/null @@ -1,143 +0,0 @@ -import pickle -import json -import numpy as np -from tqdm import tqdm -from ast import literal_eval -from glob import glob -from sentence_transformers import SentenceTransformer, util -import matplotlib.pyplot as plt -from datasets import load_dataset, Dataset, Features, Value, Sequence, DatasetDict - -from utils import * - -def embed_sentences(data, col_name, id_name, model, save_path, push_to_hub=False): - pool = model.start_multi_process_pool() - embeddings = model.encode_multi_process(data[col_name], pool=pool) - qids = data[id_name] - features = Features({id_name: Value(dtype='string'), 'embeddings': Sequence(Value('float32'))}) - embed_dict = { - id_name: qids, - "embeddings": embeddings - } - embed_ds = Dataset.from_dict(embed_dict, features=features) - if push_to_hub: - embed_ds.push_to_hub(f"bigcode/{save_path}") - else: - embed_ds.save_to_disk(save_path) - return embed_ds - - -def get_top_docs(query_embs, doc_emb, docs): - scores = np.dot(query_embs, doc_emb.T) - top_doc_indices = np.argmax(scores, axis=1) - top_scores = scores[np.arange(len(scores)), top_doc_indices] - results = [(i, docs[doc_idx], score) for i, (doc_idx, score) in tqdm(enumerate(zip(top_doc_indices, top_scores)))] - - return results - - -def filter_top_k_percent(results, k_percent): - all_scores = [score for _, score in results] - threshold = np.percentile(all_scores, 100 - k_percent) - filtered_results = [(i, doc, score) for i, doc, score in results if score > threshold] - return filtered_results - - -def filter_top_threshold(results, threshold): - filtered_results = [(i, doc, score) for i, doc, score in results if score > threshold] - return filtered_results - - -def read_task_perf(top_tid, task="complete"): - model_results = dict() - result_files = [] - for model, info in model_info.items(): - if task == "instruct" and (not info["prompted"] or info["name"] in ["Granite-Code-3B-Instruct", "Granite-Code-8B-Instruct"]): - continue - task_perf = {f"BigCodeBench/{task_id}": 0 for task_id in range(1140)} - model = model.replace("/", "--") - if info["link"].startswith("https://huggingface.co/"): - model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") - try: - if info["prompted"]: - files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") - if files: - file = files[0] - else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] - else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] - except: - continue - with open(file, "r") as f: - data = json.load(f) - for task_id, perfs in data["eval"].items(): - status = 1 if perfs[0]["status"] == "pass" else 0 - task_perf[task_id] = status - model_results[info["name"]] = np.mean([status for tid, status in task_perf.items() if tid in top_tid]) - return sorted(model_results.items(), key=lambda x: x[1], reverse=True) - - -if __name__ == "__main__": - bcb = load_dataset("bigcode/bigcodebench", trust_remote_code=True, split="v0.1.0_hf") - se = load_dataset("bigcode/stack-exchange-preferences-20230914-clean-anonymization", trust_remote_code=True, split="train") - model = SentenceTransformer("sentence-transformers/all-mpnet-base-v2") - - se_embed = embed_sentences(se, "question", "qid", model, "stack-exchange-embeddings-20230914", push_to_hub=True) - bcb_embed = embed_sentences(bcb, "complete_prompt", "task_id", model, "bigcodebench-doc-embeddings", push_to_hub=True) - - solve_rate = load_dataset("bigcode/bigcodebench-solve-rate", trust_remote_code=True, split="complete") - - query_embs = np.array(se_embed["embeddings"]) - doc_emb = np.array(bcb_embed["embeddings"]) - docs = bcb_embed["task_id"] - retrieval_results = get_top_docs(query_embs, doc_emb, docs) - - Dataset.from_dict({"qid": [i for i, _, _ in retrieval_results], "tid": [doc for _, doc, _ in retrieval_results], "score": [score for _, _, score in retrieval_results]}).push_to_hub("bigcode/se_bcb_results") - - retrieval_ds = load_dataset("bigcode/se_bcb_results", trust_remote_code=True, split="train") - retrieval_ds = load_dataset("bigcode/se_bcb_instruct_results", trust_remote_code=True, split="train") - - top_results = dict() - for sample in tqdm(retrieval_ds): - i, doc, score = sample["qid"], sample["tid"], sample["score"] - if score > 0.7: - if doc not in top_results: - top_results[doc] = (i, doc, score) - else: - if score > top_results[doc][2]: - top_results[doc] = (i, doc, score) - - top_id = {task_id: (qid, score) for qid, task_id, score in top_results.values()} - - lib_filter = {sample["task_id"] for sample in bcb if len(literal_eval(sample["libs"])) > 2} - length_filter = {sample["task_id"] for sample in bcb if len(sample["canonical_solution"]) > 426} - rate_filter = {task["task_id"]: task["solve_rate"] for task in solve_rate if task["solve_rate"] < 50} - - top_tid = top_id.keys() & length_filter & rate_filter.keys() & lib_filter - # hard_results = read_task_perf(top_tid) - - filtered_bcb = bcb.filter(lambda x: x["task_id"] in top_tid) - hard_bcb_tid = hard_bcb["task_id"] - se_qid = [top_id[_id][0] for _id in hard_bcb_tid] - se_q = se.select(se_qid) - se_scores = [top_id[_id][1] for _id in hard_bcb_tid] - - hard_bcb_dict = { - "task_id": hard_bcb_tid, - "complete_prompt": hard_bcb["complete_prompt"], - "instruct_prompt": hard_bcb["instruct_prompt"], - "canonical_solution": hard_bcb["canonical_solution"], - "code_prompt": hard_bcb["code_prompt"], - "test": hard_bcb["test"], - "entry_point": hard_bcb["entry_point"], - "doc_struct": hard_bcb["doc_struct"], - "libs": hard_bcb["libs"], - "q_idx": se_qid, - "question": se_q["question"], - "score": se_scores, - "_id": hard_bcb_tid - } - - hard_bcb = Dataset.from_dict(hard_bcb_dict) - DatasetDict({"v0.1.0_hf": hard_bcb}).push_to_hub("bigcode/bigcodebench-hard") \ No newline at end of file diff --git a/analysis/utils.py b/analysis/utils.py index f954f6d..0eecf53 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -3,623 +3,864 @@ "name": "Magicoder-S-DS-6.7B", "link": "https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B", "prompted": True, + "moe": False, "size": 6.7, + "act_param": 6.7, "open-data": "Partial", }, "bigcode/starcoder2-15b-instruct-v0.1": { "name": "StarCoder2-15B-Instruct-v0.1", "link": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1", "prompted": True, + "moe": False, "size": 15, + "act_param": 15, "open-data": "Full", }, "bigcode/starcoder2-3b": { "name": "StarCoder2-3B", "link": "https://huggingface.co/bigcode/starcoder2-3b", "prompted": False, + "moe": False, "size": 3, + "act_param": 3, "open-data": "Full", }, "bigcode/starcoder2-7b": { "name": "StarCoder2-7B", "link": "https://huggingface.co/bigcode/starcoder2-7b", "prompted": False, + "moe": False, "size": 7, + "act_param": 7, "open-data": "Full", }, "bigcode/starcoder2-15b": { "name": "StarCoder2-15B", "link": "https://huggingface.co/bigcode/starcoder2-15b", "prompted": False, + "moe": False, "size": 15, + "act_param": 15, "open-data": "Full", }, "Qwen/CodeQwen1.5-7B": { "name": "CodeQwen1.5-7B", "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B", "prompted": False, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "google/codegemma-2b": { "name": "CodeGemma-2B", "link": "https://huggingface.co/google/codegemma-2b", "prompted": False, + "moe": False, "size": 2, + "act_param": 2, "open-data": "None", }, "google/codegemma-7b": { "name": "CodeGemma-7B", "link": "https://huggingface.co/google/codegemma-7b", "prompted": False, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "google/codegemma-7b-it": { "name": "CodeGemma-7B-Instruct", "link": "https://huggingface.co/google/codegemma-7b-it", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "gpt-3.5-turbo-0125": { "name": "GPT-3.5-Turbo-0125", "link": "https://openai.com/index/new-embedding-models-and-api-updates", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "gpt-4o": { "name": "GPT-4o-2024-05-13", "link": "https://openai.com/index/hello-gpt-4o/", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "gpt-4-turbo-2024-04-09": { "name": "GPT-4-Turbo-2024-04-09", "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "gpt-4-0613": { "name": "GPT-4-0613", "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "codellama/CodeLlama-7b-hf": { "name": "CodeLlama-7B-Base", "link": "https://huggingface.co/codellama/CodeLlama-7b-hf", "prompted": False, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "codellama/CodeLlama-13b-hf": { "name": "CodeLlama-13B-Base", "link": "https://huggingface.co/codellama/CodeLlama-13b-hf", "prompted": False, + "moe": False, "size": 13, + "act_param": 13, "open-data": "None", }, "codellama/CodeLlama-7b-Instruct-hf": { "name": "CodeLlama-7B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "codellama/CodeLlama-13b-Instruct-hf": { "name": "CodeLlama-13B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf", "prompted": True, + "moe": False, "size": 13, + "act_param": 13, "open-data": "None", }, "mistral-large-2402": { "name": "Mistral-Large-2402", "link": "https://mistral.ai/news/mistral-large/", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "mistral-small-2402": { "name": "Mistral-Small-2402", "link": "https://mistral.ai/news/mistral-large/", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "mistralai/Mixtral-8x22B-v0.1": { "name": "Mixtral-8x22B-Base", "link": "https://huggingface.co/mistralai/Mixtral-8x22B-v0.1", "prompted": False, - "size": 44, + "moe": True, + "size": 176, + "act_param": 44, "open-data": "None", }, "mistralai/Mixtral-8x22B-Instruct-v0.1": { "name": "Mixtral-8x22B-Instruct", "link": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1", "prompted": True, - "size": 44, + "moe": False, + "size": 176, + "act_param": 44, "open-data": "None", }, "codellama/CodeLlama-34b-hf": { "name": "CodeLlama-34B-Base", "link": "https://huggingface.co/codellama/CodeLlama-34b-hf", "prompted": False, + "moe": False, "size": 34, + "act_param": 34, "open-data": "None", }, "codellama/CodeLlama-34b-Instruct-hf": { "name": "CodeLlama-34B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf", "prompted": True, + "moe": False, "size": 34, + "act_param": 34, "open-data": "None", }, "codellama/CodeLlama-70b-hf": { "name": "CodeLlama-70B-Base", "link": "https://huggingface.co/codellama/CodeLlama-70b-hf", "prompted": False, + "moe": False, "size": 70, + "act_param": 70, "open-data": "None", }, "codellama/CodeLlama-70b-Instruct-hf": { "name": "CodeLlama-70B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf", "prompted": True, + "moe": False, "size": 70, + "act_param": 70, "open-data": "None", }, "Qwen/CodeQwen1.5-7B-Chat": { "name": "CodeQwen1.5-7B-Chat", "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "Qwen/Qwen1.5-110B-Chat": { "name": "Qwen1.5-110B-Chat", "link": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat", "prompted": True, + "moe": False, "size": 110, + "act_param": 110, "open-data": "None", }, "Qwen/Qwen1.5-72B-Chat": { "name": "Qwen1.5-72B-Chat", "link": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat", "prompted": True, + "moe": False, "size": 72, + "act_param": 72, "open-data": "None", }, "Qwen/Qwen1.5-32B-Chat": { "name": "Qwen1.5-32B-Chat", "link": "https://huggingface.co/Qwen/Qwen1.5-32B-Chat", "prompted": True, + "moe": False, "size": 32, + "act_param": 32, "open-data": "None", }, "deepseek-ai/DeepSeek-V2-Chat": { "name": "DeepSeek-V2-Chat", - "link": "https://www.deepseek.com/", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat", "prompted": True, - "size": 21, + "moe": True, + "size": 236, + "act_param": 21, "open-data": "None", }, "deepseek-ai/deepseek-coder-1.3b-base": { "name": "DeepSeek-Coder-1.3B-Base", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base", "prompted": False, + "moe": False, "size": 1.3, + "act_param": 1.3, "open-data": "None", }, "deepseek-ai/deepseek-coder-1.3b-instruct": { "name": "DeepSeek-Coder-1.3B-Instruct", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct", "prompted": True, + "moe": False, "size": 1.3, + "act_param": 1.3, "open-data": "None", }, "deepseek-ai/deepseek-coder-33b-base": { "name": "DeepSeek-Coder-33B-Base", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base", "prompted": False, + "moe": False, "size": 33, + "act_param": 33, "open-data": "None", }, "deepseek-ai/deepseek-coder-33b-instruct": { "name": "DeepSeek-Coder-33B-Instruct", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct", "prompted": True, + "moe": False, "size": 33, + "act_param": 33, "open-data": "None", }, "deepseek-ai/deepseek-coder-6.7b-base": { "name": "DeepSeek-Coder-6.7B-Base", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", "prompted": False, + "moe": False, "size": 6.7, + "act_param": 6.7, "open-data": "None", }, "deepseek-ai/deepseek-coder-6.7b-instruct": { "name": "DeepSeek-Coder-6.7B-Instruct", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct", "prompted": True, + "moe": False, "size": 6.7, + "act_param": 6.7, "open-data": "None", }, "meta-llama/Meta-Llama-3-70B": { "name": "Llama-3-70B-Base", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B", "prompted": False, + "moe": False, "size": 70, + "act_param": 70, "open-data": "None", }, "meta-llama/Meta-Llama-3-70B-Instruct": { "name": "Llama-3-70B-Instruct", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct", "prompted": True, + "moe": False, "size": 70, + "act_param": 70, "open-data": "None", }, "meta-llama/Meta-Llama-3-8B": { "name": "Llama-3-8B-Base", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", "prompted": False, + "moe": False, "size": 8, + "act_param": 8, "open-data": "None", }, "meta-llama/Meta-Llama-3-8B-Instruct": { "name": "Llama-3-8B-Instruct", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct", "prompted": True, + "moe": False, "size": 8, + "act_param": 8, "open-data": "None", }, "ibm-granite/granite-3b-code-instruct": { "name": "Granite-Code-3B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-3b-code-instruct", "prompted": True, + "moe": False, "size": 3, + "act_param": 3, "open-data": "None", }, "ibm-granite/granite-8b-code-instruct": { "name": "Granite-Code-8B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-8b-code-instruct", "prompted": True, + "moe": False, "size": 8, + "act_param": 8, "open-data": "None", }, "ibm-granite/granite-20b-code-instruct": { "name": "Granite-Code-20B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-20b-code-instruct", "prompted": True, + "moe": False, "size": 20, + "act_param": 20, "open-data": "None", }, "ibm-granite/granite-34b-code-instruct": { "name": "Granite-Code-34B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-34b-code-instruct", "prompted": True, + "moe": False, "size": 34, + "act_param": 34, "open-data": "None", }, "ibm-granite/granite-3b-code-base": { "name": "Granite-Code-3B-Base", "link": "https://huggingface.co/ibm-granite/granite-3b-code-base", "prompted": False, + "moe": False, "size": 3, + "act_param": 3, "open-data": "None", }, "ibm-granite/granite-8b-code-base": { "name": "Granite-Code-8B-Base", "link": "https://huggingface.co/ibm-granite/granite-8b-code-base", "prompted": False, + "moe": False, "size": 8, + "act_param": 8, "open-data": "None", }, "ibm-granite/granite-20b-code-base": { "name": "Granite-Code-20B-Base", "link": "https://huggingface.co/ibm-granite/granite-20b-code-base", "prompted": False, + "moe": False, "size": 20, + "act_param": 20, "open-data": "None", }, "ibm-granite/granite-34b-code-base": { "name": "Granite-Code-34B-Base", "link": "https://huggingface.co/ibm-granite/granite-34b-code-base", "prompted": False, + "moe": False, "size": 34, + "act_param": 34, "open-data": "None", }, "claude-3-haiku-20240307": { "name": "Claude-3-Haiku-20240307", "link": "https://www.anthropic.com/news/claude-3-family", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "claude-3-sonnet-20240229": { "name": "Claude-3-Sonnet-20240229", "link": "https://www.anthropic.com/news/claude-3-family", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "claude-3-opus-20240229": { "name": "Claude-3-Opus-20240229", "link": "https://www.anthropic.com/news/claude-3-family", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "01-ai/Yi-1.5-34B-Chat": { "name": "Yi-1.5-34B-Chat", "link": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat", "prompted": True, + "moe": False, "size": 34, + "act_param": 34, "open-data": "None", }, "01-ai/Yi-1.5-34B": { "name": "Yi-1.5-34B", "link": "https://huggingface.co/01-ai/Yi-1.5-34B", "prompted": False, + "moe": False, "size": 34, + "act_param": 34, "open-data": "None", }, "01-ai/Yi-1.5-9B-Chat": { "name": "Yi-1.5-9B-Chat", "link": "https://huggingface.co/01-ai/Yi-1.5-9B-Chat", "prompted": True, + "moe": False, "size": 9, + "act_param": 9, "open-data": "None", }, "01-ai/Yi-1.5-9B": { "name": "Yi-1.5-9B", "link": "https://huggingface.co/01-ai/Yi-1.5-9B", "prompted": False, + "moe": False, "size": 9, + "act_param": 9, "open-data": "None", }, "01-ai/Yi-1.5-6B-Chat": { "name": "Yi-1.5-6B-Chat", "link": "https://huggingface.co/01-ai/Yi-1.5-6B-Chat", "prompted": True, - "size": 9, + "moe": False, + "size": 6, + "act_param": 6, "open-data": "None", }, "01-ai/Yi-1.5-6B": { "name": "Yi-1.5-6B", "link": "https://huggingface.co/01-ai/Yi-1.5-6B", "prompted": False, - "size": 9, + "moe": False, + "size": 6, + "act_param": 6, "open-data": "None", }, "Qwen/Qwen2-57B-A14B": { "name": "Qwen2-57B-A14B", "link": "https://huggingface.co/Qwen/Qwen2-57B-A14B", "prompted": True, - "size": 14, + "moe": True, + "size": 57, + "act_param": 14, "open-data": "None", }, "Qwen/Qwen2-7B-Instruct": { "name": "Qwen2-7B-Instruct", "link": "https://huggingface.co/Qwen/Qwen2-7B-Instruct", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "Qwen/Qwen2-72B-Chat": { "name": "Qwen2-72B-Chat", "link": "https://huggingface.co/Qwen/Qwen2-72B-Chat", "prompted": True, + "moe": False, "size": 72, + "act_param": 72, "open-data": "None", }, "gemini-1.5-pro": { "name": "Gemini-1.5-Pro-API-0514", "link": "https://deepmind.google/technologies/gemini/pro", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "gemini-1.5-flash": { "name": "Gemini-1.5-Flash-API-0514", "link": "https://deepmind.google/technologies/gemini/flash/", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "m-a-p/OpenCodeInterpreter-DS-33B": { "name": "OpenCodeInterpreter-DS-33B", "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B", "prompted": True, + "moe": False, "size": 33, + "act_param": 33, "open-data": "Partial", }, "m-a-p/OpenCodeInterpreter-DS-6.7B": { "name": "OpenCodeInterpreter-DS-6.7B", "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B", "prompted": True, + "moe": False, "size": 6.7, + "act_param": 6.7, "open-data": "Partial", }, "m-a-p/OpenCodeInterpreter-DS-1.3B": { "name": "OpenCodeInterpreter-DS-1.3B", "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B", "prompted": True, + "moe": False, "size": 1.3, + "act_param": 1.3, "open-data": "Partial", }, "microsoft/Phi-3-medium-128k-instruct": { "name": "Phi-3-Medium-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct", "prompted": True, + "moe": False, "size": 14, + "act_param": 14, "open-data": "None", }, "microsoft/Phi-3-small-128k-instruct": { "name": "Phi-3-Small-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "mistralai/Codestral-22B-v0.1": { "name": "Codestral-22B-v0.1", "link": "https://huggingface.co/mistralai/Codestral-22B-v0.1", "prompted": True, + "moe": False, "size": 22, + "act_param": 22, "open-data": "None", }, "mistralai/Mistral-7B-Instruct-v0.3": { "name": "Mistral-7B-Instruct-v0.3", "link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "mistralai/Mistral-7B-v0.3": { "name": "Mistral-7B-v0.3", "link": "https://huggingface.co/mistralai/Mistral-7B-v0.3", "prompted": False, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "CohereForAI/c4ai-command-r-plus": { "name": "Command R+", "link": "https://huggingface.co/CohereForAI/c4ai-command-r-plus", "prompted": True, + "moe": False, "size": 104, + "act_param": 104, "open-data": "None", }, "deepseek-coder": { "name": "DeepSeek-Coder-V2-Instruct", "link": "https://www.deepseek.com/", "prompted": True, - "size": 21, + "moe": True, + "size": 236, + "act_param": 21, "open-data": "None", }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": { "name": "DeepSeek-Coder-V2-Lite-Instruct", "link": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct", "prompted": True, - "size": 2.4, + "moe": True, + "size": 16, + "act_param": 2.4, "open-data": "None", }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Base": { "name": "DeepSeek-Coder-V2-Lite-Base", "link": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Base", "prompted": False, - "size": 2.4, + "moe": True, + "size": 16, + "act_param": 2.4, "open-data": "None", }, "claude-3-5-sonnet-20240620": { "name": "Claude-3.5-Sonnet-20240620", "link": "https://claude.ai/", "prompted": True, + "moe": False, "size": None, + "act_param": None, "open-data": "None", }, "NousResearch/Hermes-2-Theta-Llama-3-70B": { "name": "Hermes-2-Theta-Llama-3-70B", "link": "https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-70B", "prompted": True, + "moe": False, "size": 70, + "act_param": 70, "open-data": "None", }, "microsoft/wavecoder-ultra-6.7b": { "name": "WaveCoder-Ultra-6.7B", "link": "https://huggingface.co/microsoft/wavecoder-ultra-6.7b", "prompted": True, + "moe": False, "size": 6.7, + "act_param": 6.7, "open-data": "None", }, "google/gemma-2-9b-it": { "name": "Gemma-2-9B-Instruct", "link": "https://huggingface.co/google/gemma-2-9b-it", "prompted": True, + "moe": False, "size": 9, + "act_param": 9, "open-data": "None", }, "Bin12345/AutoCoder": { "name": "AutoCoder", "link": "https://huggingface.co/Bin12345/AutoCoder", "prompted": True, + "moe": False, "size": 33, + "act_param": 33, "open-data": "None", }, "Bin12345/AutoCoder_S_6.7B": { "name": "AutoCoder-S-6.7B", "link": "https://huggingface.co/Bin12345/AutoCoder_S_6.7B", "prompted": True, + "moe": False, "size": 6.7, + "act_param": 6.7, "open-data": "None", }, "Bin12345/AutoCoder_QW_7B": { "name": "AutoCoder-QW-7B", "link": "https://huggingface.co/Bin12345/AutoCoder_QW_7B", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "SenseLLM/ReflectionCoder-DS-33B": { "name": "ReflectionCoder-DS-33B", "link": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-33B", "prompted": True, + "moe": False, "size": 33, + "act_param": 33, "open-data": "Partial", }, "SenseLLM/ReflectionCoder-DS-6.7B": { "name": "ReflectionCoder-DS-6.7B", "link": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-6.7B", "prompted": True, + "moe": False, "size": 6.7, + "act_param": 6.7, "open-data": "Partial", }, "SenseLLM/ReflectionCoder-CL-34B": { "name": "ReflectionCoder-CL-34B", "link": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-34B", "prompted": True, + "moe": False, "size": 34, + "act_param": 34, "open-data": "Partial", }, "SenseLLM/ReflectionCoder-CL-7B": { "name": "ReflectionCoder-CL-7B", "link": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-7B", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "Partial", }, "new-microsoft/Phi-3-mini-128k-instruct": { "name": "Phi-3-Mini-128K-Instruct (June 2024)", "link": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct", "prompted": True, + "moe": False, "size": 3.8, + "act_param": 3.8, "open-data": "None", }, "old-microsoft/Phi-3-mini-128k-instruct": { "name": "Phi-3-Mini-128K-Instruct (Old)", "link": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct", "prompted": True, + "moe": False, "size": 3.8, + "act_param": 3.8, "open-data": "None", }, "internlm/internlm2_5-7b-chat": { "name": "InternLM2.5-7B-Chat", "link": "https://huggingface.co/internlm/internlm2_5-7b-chat", "prompted": True, + "moe": False, "size": 7, + "act_param": 7, "open-data": "None", }, "NousResearch/Hermes-2-Pro-Llama-3-70B": { "name": "Hermes-2-Pro-Llama-3-70B", "link": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-70B", "prompted": True, + "moe": False, "size": 70, + "act_param": 70, "open-data": "Partial", }, + "new-deepseek-chat": { + "name": "DeepSeek-V2-Chat (2024-06-28)", + "link": "https://www.deepseek.com/", + "prompted": True, + "moe": True, + "size": 236, + "act_param": 21, + "open-data": "None", + }, + "vllm-google/gemma-2-27b-it": { + "name": "Gemma-2-27B-Instruct", + "link": "https://huggingface.co/google/gemma-2-27b-it", + "prompted": True, + "moe": False, + "size": 27, + "act_param": 27, + "open-data": "None", + }, + "Artigenz/Artigenz-Coder-DS-6.7B": { + "name": "Artigenz-Coder-DS-6.7B", + "link": "https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B", + "prompted": True, + "moe": False, + "size": 6.7, + "act_param": 6.7, + "open-data": "None", + }, + "openchat/openchat-3.6-8b-20240522": { + "name": "OpenChat-3.6-8B-20240522", + "link": "https://huggingface.co/openchat/openchat-3.6-8b-20240522", + "prompted": True, + "moe": False, + "size": 8, + "act_param": 8, + "open-data": "None", + }, + "Phind/Phind-CodeLlama-34B-v2": { + "name": "Phind-CodeLlama-34B-v2", + "link": "https://huggingface.co/Phind/Phind-CodeLlama-34B-v2", + "prompted": True, + "moe": False, + "size": 34, + "act_param": 34, + "open-data": "None", + }, + "yi-large": { + "name": "Yi-Large", + "link": "https://www.lingyiwanwu.com/", + "prompted": False, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "THUDM/codegeex4-all-9b": { + "name": "CodeGeex4-All-9B", + "link": "https://huggingface.co/THUDM/codegeex4-all-9b", + "prompted": False, + "moe": False, + "size": 9, + "act_param": 9, + "open-data": "None", + }, } \ No newline at end of file From 087c202df0f2e399e3922d4a2e3c1cdd80df4277 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 15 Jul 2024 06:55:07 +0800 Subject: [PATCH 065/325] update he result analysis --- analysis/get_results.py | 159 ++++++++++++++++++++++++---------------- 1 file changed, 96 insertions(+), 63 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index 4921931..196ba26 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -9,10 +9,9 @@ import pandas as pd import itertools import math -from datasets import Dataset, DatasetDict +from datasets import Dataset, DatasetDict, load_dataset from transformers import AutoTokenizer - def update_model_info(model_info): for model, info in model_info.items(): if "https://huggingface.co/" in info["link"]: @@ -29,7 +28,7 @@ def update_model_info(model_info): return model_info -def get_results(): +def get_results(tids): results = {} for model, info in model_info.items(): results[info["name"]] = { @@ -42,7 +41,9 @@ def get_results(): "instruct-cal": None, }, "prompted": info["prompted"], + "moe": info["moe"], "size": info["size"], + "act_param": info["act_param"], "direct_complete": info["direct_complete"], } @@ -59,9 +60,9 @@ def get_results(): status = [] with open("results/"+model+"--bigcodebench-"+suffix, "r") as f: data = json.load(f) - if len(data["eval"]) != 1140: - continue for key, value in data["eval"].items(): + if key not in tids: + continue if value[0]["status"] == "pass": status.append(1) else: @@ -143,14 +144,14 @@ def split_gen(): f.writelines(data) -def read_task_perf(task="complete"): +def read_task_perf(tids, task="complete"): model_results = dict() result_files = [] for model, info in model_info.items(): if task == "instruct" and (not info["prompted"] or info["name"] in ["Granite-Code-3B-Instruct", "Granite-Code-8B-Instruct"]): continue - task_perf = {f"BigCodeBench/{task_id}": 0 for task_id in range(1140)} + task_perf = dict() model = model.replace("/", "--") # if info["link"].startswith("https://huggingface.co/"): # model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") @@ -170,13 +171,14 @@ def read_task_perf(task="complete"): with open(file, "r") as f: data = json.load(f) for task_id, perfs in data["eval"].items(): - status = 1 if perfs[0]["status"] == "pass" else 0 - task_perf[task_id] = status + if task_id in tids: + status = 1 if perfs[0]["status"] == "pass" else 0 + task_perf[task_id] = status model_results[info["name"]] = task_perf return model_results, result_files -def get_winner_df(data_dict, task, task_level=True, no_tie=True): +def get_winner_df(data_dict, tids, task, task_level=True, no_tie=True): winner_dict = {"task_id": [], "model_a": [], "model_b": [], "winner": []} if not task_level: file = f"{task}_winner_df.csv" @@ -184,8 +186,7 @@ def get_winner_df(data_dict, task, task_level=True, no_tie=True): file = f"{task}_winner_task_df.csv" if task_level: - for task_id in tqdm(range(1140)): - task_id = f"BigCodeBench/{task_id}" + for task_id in tqdm(tids): # pair without repetition (a, b) and (b, a) are the same for model_a, model_b in itertools.combinations(data_dict.keys(), 2): solve_rate_a = data_dict[model_a][task_id] @@ -264,23 +265,51 @@ def update_elo_rating(results, elo_dict): return results +def get_domain_perf(data_dict, task2domain): + domain_perfs = { + "Model": [], + "Computation": [], + "General": [], + "Visualization": [], + "System": [], + "Time": [], + "Network": [], + "Cryptography": [] + } + for model, task_perf in data_dict.items(): + model_domain = {"Computation": [], "General": [], "Visualization": [], "System": [], "Time": [], "Network": [], "Cryptography": []} + for task_id, status in task_perf.items(): + domains = task2domain[task_id] + for domain in domains: + model_domain[domain].append(status) + domain_perf = {domain: round(np.mean(perfs)*100, 1) for domain, perfs in model_domain.items()} + domain_perfs["Model"].append(model) + for domain in model_domain.keys(): + domain_perfs[domain].append(domain_perf[domain]) + return Dataset.from_dict(domain_perfs) + + def get_solve_rate(data_dict, task="complete"): - task_solve_count = {f"BigCodeBench/{task_id}": [] for task_id in range(1140)} + task_solve_count = dict() for model, task_perf in data_dict.items(): - for task_id in range(1140): - task_solve_count[f"BigCodeBench/{task_id}"].append(task_perf[f"BigCodeBench/{task_id}"]) + for task_id, score in task_perf.items(): + if task_id not in task_solve_count: + task_solve_count[task_id] = [] + task_solve_count[task_id].append(score) solve_rate = {task_id: round(np.mean(perfs) * 100, 1) for task_id, perfs in task_solve_count.items()} return Dataset.from_dict({"task_id": list(solve_rate.keys()), "solve_rate": list(solve_rate.values())}) def get_hf_ds(results): - hf_dataset = {"model": [], "link": [], "size": [], "type": [], "lazy": [], "direct_complete": [], + hf_dataset = {"model": [], "link": [], "moe": [], "size": [], "act_param": [], "type": [], "lazy": [], "direct_complete": [], "complete": [], "instruct": [], "elo_mle": []} for model, result in results.items(): hf_dataset["model"].append(model) hf_dataset["link"].append(result["link"]) + hf_dataset["moe"].append(result["moe"]) hf_dataset["size"].append(result["size"]) + hf_dataset["act_param"].append(result["act_param"]) hf_dataset["type"].append("🔶" if result["prompted"] else "🟢") hf_dataset["lazy"].append(result["lazy"]) hf_dataset["complete"].append(result["pass@1"]["complete"]) @@ -311,52 +340,56 @@ def push_ds(ds, path, local=False): if __name__ == "__main__": + bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.0_hf") + bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.0_hf") model_info = update_model_info(model_info) - results = get_results() - files = [] - complete_data, complete_files = read_task_perf("complete") - instruct_data, instruct_files = read_task_perf("instruct") - assert len(model_info) == len(complete_data) - # complete_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): - # Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in complete_data.items()} - # instruct_map = {model.replace("-","_").replace("+","_plus").replace(" ","_"): - # Dataset.from_dict({"task_id": list(task_perf.keys()), "status": list(task_perf.values())}) for model, task_perf in instruct_data.items()} - # complete_ds = DatasetDict(complete_map) - # instruct_ds = DatasetDict(instruct_map) - # push_ds(complete_ds, "bigcode/bigcodebench-complete-perf") - # push_ds(instruct_ds, "bigcode/bigcodebench-instruct-perf") - - files.extend(complete_files) - files.extend(instruct_files) - shutil.rmtree("eval_results", ignore_errors=True) - os.makedirs("eval_results", exist_ok=True) - for file in files: - shutil.copy(file, "eval_results") - - complete_solve_rate = get_solve_rate(complete_data, task="complete") - instruct_solve_rate = get_solve_rate(instruct_data, task="instruct") - solve_rate_ds = DatasetDict({"complete": complete_solve_rate, "instruct": instruct_solve_rate}) - push_ds(solve_rate_ds, "bigcode/bigcodebench-solve-rate") - - elo_config = { - "task_no_tie": (True, True), - "benchmark_tie": (False, False), + bcb_config = { + "": bcb_orig, + "-hard": bcb_hard, } - elo_ds = dict() - for config, (task_level, no_tie) in elo_config.items(): - battles = get_winner_df(complete_data, "complete", task_level=task_level, no_tie=no_tie) - elo_mle_bootstrap = get_bootstrap_result(battles, get_elo_mle, 500) - bootstrap_lu_median = elo_mle_bootstrap.median().reset_index().set_axis(["model", "Elo rating"], axis=1) - bootstrap_lu_median["Elo rating"] = (bootstrap_lu_median["Elo rating"] + 0.5).astype(int) - bootstrap_lu_median_dict = bootstrap_lu_median.set_index("model")["Elo rating"].to_dict() - if config == "task_no_tie": - task_elo = bootstrap_lu_median_dict - elo = get_bootstrap_scores(elo_mle_bootstrap) - elo_ds[config] = elo - push_ds(DatasetDict(elo_ds), "bigcode/bigcodebench-elo") - - results = update_elo_rating(results, task_elo) - with open("results.json", "w") as f: - json.dump(results, f, indent=4) - ds = get_hf_ds(results) - push_ds(ds, "bigcode/bigcodebench-results") \ No newline at end of file + for suffix, bcb in bcb_config.items(): + results = get_results(bcb["task_id"]) + files = [] + complete_data, complete_files = read_task_perf(bcb["task_id"], "complete") + instruct_data, instruct_files = read_task_perf(bcb["task_id"], "instruct") + assert len(model_info) == len(complete_data) + with open("task2domain.json", "r") as f: + task2domain = json.load(f) + domain_complete = get_domain_perf(complete_data, task2domain) + domain_instruct = get_domain_perf(instruct_data, task2domain) + DatasetDict({"complete": domain_complete, "instruct": domain_instruct}).push_to_hub(f"bigcode/bigcodebench{suffix}-domain") + + files.extend(complete_files) + files.extend(instruct_files) + shutil.rmtree("eval_results", ignore_errors=True) + os.makedirs("eval_results", exist_ok=True) + for file in files: + shutil.copy(file, "eval_results") + + complete_solve_rate = get_solve_rate(complete_data, task="complete") + instruct_solve_rate = get_solve_rate(instruct_data, task="instruct") + solve_rate_ds = DatasetDict({"complete": complete_solve_rate, "instruct": instruct_solve_rate}) + push_ds(solve_rate_ds, f"bigcode/bigcodebench{suffix}-solve-rate") + + elo_config = { + "task_no_tie": (True, True), + "benchmark_tie": (False, False), + } + elo_ds = dict() + for config, (task_level, no_tie) in elo_config.items(): + battles = get_winner_df(complete_data, bcb["task_id"], "complete", task_level=task_level, no_tie=no_tie) + elo_mle_bootstrap = get_bootstrap_result(battles, get_elo_mle, 500) + bootstrap_lu_median = elo_mle_bootstrap.median().reset_index().set_axis(["model", "Elo rating"], axis=1) + bootstrap_lu_median["Elo rating"] = (bootstrap_lu_median["Elo rating"] + 0.5).astype(int) + bootstrap_lu_median_dict = bootstrap_lu_median.set_index("model")["Elo rating"].to_dict() + if config == "task_no_tie": + task_elo = bootstrap_lu_median_dict + elo = get_bootstrap_scores(elo_mle_bootstrap) + elo_ds[config] = elo + push_ds(DatasetDict(elo_ds), f"bigcode/bigcodebench{suffix}-elo") + + results = update_elo_rating(results, task_elo) + with open(f"results{suffix}.json", "w") as f: + json.dump(results, f, indent=4) + ds = get_hf_ds(results) + push_ds(ds, f"bigcode/bigcodebench{suffix}-results") \ No newline at end of file From 3978502fda80ee32eccb8da8e7cb9377b5bba961 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 15 Jul 2024 09:42:13 +0800 Subject: [PATCH 066/325] update models --- analysis/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index 0eecf53..e78d8c2 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -848,7 +848,7 @@ "yi-large": { "name": "Yi-Large", "link": "https://www.lingyiwanwu.com/", - "prompted": False, + "prompted": True, "moe": False, "size": None, "act_param": None, @@ -857,7 +857,7 @@ "THUDM/codegeex4-all-9b": { "name": "CodeGeex4-All-9B", "link": "https://huggingface.co/THUDM/codegeex4-all-9b", - "prompted": False, + "prompted": True, "moe": False, "size": 9, "act_param": 9, From ef09c424965066d48ebae2cb2218e0114456d41f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 16 Jul 2024 16:00:38 +0800 Subject: [PATCH 067/325] doc: update for v0.1.8 --- README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 241116b..814cc37 100755 --- a/README.md +++ b/README.md @@ -104,10 +104,12 @@ pip install -U flash-attn To generate code samples from a model, you can use the following command: > ```bash +# when greedy, there is no need for temperature and n_samples bigcodebench.generate \ --model [model_name] \ - --subset [complete|instruct] \ - --greedy \ + --split [complete|instruct] \ + --subset [full|hard] \ + [--greedy] \ --bs [bs] \ --temperature [temp] \ --n_samples [n_samples] \ @@ -124,7 +126,8 @@ The generated code samples will be stored in a file named `[model_name]--bigcode # If you are using GPUs docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ --model [model_name] \ - --subset [complete|instruct] \ + --split [complete|instruct] \ + --subset [full|hard] \ [--greedy] \ --bs [bs] \ --temperature [temp] \ @@ -136,7 +139,8 @@ docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/app -t bigcodebenc # ...Or if you are using CPUs docker run -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ --model [model_name] \ - --subset [complete|instruct] \ + --split [complete|instruct] \ + --subset [full|hard] \ [--greedy] \ --bs [bs] \ --temperature [temp] \ @@ -233,10 +237,10 @@ You are strongly recommended to use a sandbox such as [docker](https://docs.dock # If you want to change the RAM address space limit (in MB, 128 GB by default): `--max-as-limit XXX` # If you want to change the RAM data segment limit (in MB, 4 GB by default): `--max-data-limit` # If you want to change the RAM stack limit (in MB, 4 MB by default): `--max-stack-limit` -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --subset [complete|instruct] --samples samples-sanitized-calibrated.jsonl +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl # If you only want to check the ground truths -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --subset [complete|instruct] --samples samples-sanitized-calibrated.jsonl --check-gt-only +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --check-gt-only ``` ...Or if you want to try it locally regardless of the risks ⚠️: @@ -251,12 +255,12 @@ Then, run the evaluation: ```bash # ...Or locally ⚠️ -bigcodebench.evaluate --subset [complete|instruct] --samples samples-sanitized-calibrated.jsonl +bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl # ...If you really don't want to check the ground truths -bigcodebench.evaluate --subset [complete|instruct] --samples samples-sanitized-calibrated.jsonl --no-gt +bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt # You are strongly recommended to use the following command to clean up the environment after evaluation: -pids=$(ps -u $(id -u) -o pid,comm | grep '^ *[0-9]\\+ bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; +pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/* ``` From af94e978e94776bb5a78b6250088f659c2d733af Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 16 Jul 2024 17:17:01 +0800 Subject: [PATCH 068/325] fix: use legacy=False --- bigcodebench/model.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/bigcodebench/model.py b/bigcodebench/model.py index 19110d5..4fb7529 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -26,7 +26,6 @@ warn("GoogleGenAI decoder will not work. Fix by `pip install google-generativeai`") import torch -from stop_sequencer import StopSequencer from transformers import AutoModelForCausalLM, AutoTokenizer try: @@ -130,10 +129,11 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: "trust_remote_code": self.trust_remote_code, } - self.tokenizer = AutoTokenizer.from_pretrained(self.name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.name, legacy=False, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) self.llm = LLM(model=name, max_model_len=2048, **kwargs) + self.llm.set_tokenizer(tokenizer=self.tokenizer) def is_direct_completion(self) -> bool: return self.tokenizer.chat_template is None @@ -179,7 +179,7 @@ def __init__(self, name: str, dataset: str, **kwargs): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") kwargs = {} - kwargs["device_map"] = "auto" + kwargs["device_map"] = "cuda:0" kwargs["trust_remote_code"] = self.trust_remote_code # string to torch dtype kwargs["torch_dtype"] = getattr(torch, self.dtype) @@ -187,7 +187,7 @@ def __init__(self, name: str, dataset: str, **kwargs): print(f"{kwargs = }") - self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(name, legacy=False, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) @@ -213,18 +213,7 @@ def codegen( kwargs["top_p"] = 0.95 kwargs["temperature"] = self.temperature - stop_sequencer = StopSequencer( - self.model, - model_type="causal", # or seq2seq - tokenizer=self.tokenizer, - ) - - model = stop_sequencer.register_stop_texts( - stop_texts=self.eos, - input_length=input_tokens.size(-1), - ) - - outputs = model.generate( + outputs = self.model.generate( input_tokens, max_new_tokens=self.max_new_tokens, do_sample=do_sample, @@ -253,7 +242,7 @@ def __init__(self, name: str, **kwargs): super().__init__(name=name, **kwargs) self.eos += ["\n```\n"] print(f"EOS strings: {self.eos}") - self.tokenizer = AutoTokenizer.from_pretrained(self.name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.name, legacy=False, **kwargs) def codegen( self, prompt: str, do_sample: bool = True, num_samples: int = 200 From cc814a3253f70e5d8958aa9fbf2850a56884be8a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 16 Jul 2024 17:18:31 +0800 Subject: [PATCH 069/325] fix: remove stop seq --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index af44b9b..6f1c731 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,6 @@ generate = anthropic>=0.26.1 google-generativeai>=0.5.4 mistralai>=0.2.0 - stop-sequencer>=1.2.3 openai>=1.11.1 [options.entry_points] From b4d27580c2dd2bcb4a1b84350c0e10ae793b8344 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 16 Jul 2024 17:25:40 +0800 Subject: [PATCH 070/325] fix: update run example --- run.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/run.sh b/run.sh index 1177728..4b72f85 100755 --- a/run.sh +++ b/run.sh @@ -5,7 +5,8 @@ BACKEND=openai TEMP=0 N_SAMPLES=1 NUM_GPU=1 -SUBSET=instruct +SPLIT=complete +SUBSET=hard if [[ $MODEL == *"/"* ]]; then ORG=$(echo $MODEL | cut -d'/' -f1)-- BASE_MODEL=$(echo $MODEL | cut -d'/' -f2) @@ -18,20 +19,21 @@ FILE_HEADER=$ORG$BASE_MODEL--$DATASET-$SUBSET--$BACKEND-$TEMP-$N_SAMPLES echo $FILE_HEADER bigcodebench.generate \ - --id_range 0 1 \ --tp $NUM_GPU \ --model $MODEL \ --bs $BS \ --temperature $TEMP \ --n_samples $N_SAMPLES \ --resume \ + --split $SPLIT \ --subset $SUBSET \ - --backend $BACKEND + --backend $BACKEND \ + --trust_remote_code bigcodebench.sanitize --samples $FILE_HEADER.jsonl --calibrate # Check if the ground truth works on your machine -bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl +bigcodebench.evaluate --split $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl # If the execution is slow: -bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl --parallel 32 \ No newline at end of file +bigcodebench.evaluate --split $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl --parallel 32 \ No newline at end of file From bbc070e160b6ea854a4f49342f9e1df492507a46 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 16 Jul 2024 17:32:00 +0800 Subject: [PATCH 071/325] fix device --- bigcodebench/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/model.py b/bigcodebench/model.py index 4fb7529..77e4d36 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -179,7 +179,7 @@ def __init__(self, name: str, dataset: str, **kwargs): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") kwargs = {} - kwargs["device_map"] = "cuda:0" + kwargs["device_map"] = "auto" kwargs["trust_remote_code"] = self.trust_remote_code # string to torch dtype kwargs["torch_dtype"] = getattr(torch, self.dtype) From 133b067466088c71351fbaee6adbf21c157d4aa1 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 04:41:49 +0800 Subject: [PATCH 072/325] feat: add customized tokenizer --- bigcodebench/evaluate.py | 26 ++++++++++++++++++++++++++ bigcodebench/generate.py | 4 +++- bigcodebench/model.py | 21 +++++++++++++++------ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 59ad079..f7fc31f 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -281,6 +281,32 @@ def stucking_checker(): if not os.path.isfile(result_path): with open(result_path, "w") as f: json.dump(results, f, indent=2) + + pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") + pass_at_k["model"] = flags.samples.split("/")[-1].replace(".jsonl", "") + pass_at_k["subset"] = flags.subset + + def save_pass_at_k(): + with open(pass_at_k_path, "w") as f: + json.dump(pass_at_k, f, indent=2) + + if os.path.isfile(pass_at_k_path): + saved_pass_at_k = json.load(open(pass_at_k_path, "r")) + # compare saved_pass_at_k with pass_at_k + for k in saved_pass_at_k.keys(): + if pass_at_k[k] != saved_pass_at_k[k]: + cprint(f"Warning: {k} is different from the saved one", "yellow") + + # ask user whether to save the pass@k + decision = "" + while decision.lower() not in ["y", "n"]: + print(f"Save pass@k to {pass_at_k_path}? [Y/N]") + decision = input() + if decision.lower() == "y": + save_pass_at_k() + + else: + save_pass_at_k() def main(): diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 6f9c35e..cc3d3b4 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -119,6 +119,7 @@ def main(): parser.add_argument("--base_url", default=None, type=str) parser.add_argument("--tp", default=1, type=int) parser.add_argument("--trust_remote_code", action="store_true") + parser.add_argument("--tokenizer_name", default=None, type=str) args = parser.parse_args() if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\ @@ -142,7 +143,8 @@ def main(): temperature=args.temperature, base_url=args.base_url, tp=args.tp, - trust_remote_code=args.trust_remote_code + trust_remote_code=args.trust_remote_code, + tokenizer_name=args.tokenizer_name ) extra = "-" + args.subset if args.subset != "full" else "" diff --git a/bigcodebench/model.py b/bigcodebench/model.py index 77e4d36..2740019 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -91,6 +91,7 @@ def __init__( max_new_tokens: int = 1280, dtype: str = "bfloat16", # default trust_remote_code: bool = False, + tokenizer_name: str = None, ) -> None: print("Initializing a decoder model: {} ...".format(name)) self.name = name @@ -101,6 +102,7 @@ def __init__( self.max_new_tokens = max_new_tokens self.dtype = dtype self.trust_remote_code = trust_remote_code + self.tokenizer_name = tokenizer_name @abstractmethod def codegen( @@ -128,8 +130,10 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: "dtype": self.dtype, "trust_remote_code": self.trust_remote_code, } - - self.tokenizer = AutoTokenizer.from_pretrained(self.name, legacy=False, **kwargs) + if self.tokenizer_name is None: + self.tokenizer_name = self.name + + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, legacy=False, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) self.llm = LLM(model=name, max_model_len=2048, **kwargs) @@ -185,9 +189,11 @@ def __init__(self, name: str, dataset: str, **kwargs): kwargs["torch_dtype"] = getattr(torch, self.dtype) self.skip_special_tokens = True - print(f"{kwargs = }") - - self.tokenizer = AutoTokenizer.from_pretrained(name, legacy=False, **kwargs) + print(f"{kwargs = }", self.tokenizer_name) + if self.tokenizer_name is None: + self.tokenizer_name = self.name + + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, legacy=False, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) @@ -242,7 +248,7 @@ def __init__(self, name: str, **kwargs): super().__init__(name=name, **kwargs) self.eos += ["\n```\n"] print(f"EOS strings: {self.eos}") - self.tokenizer = AutoTokenizer.from_pretrained(self.name, legacy=False, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name, **kwargs) def codegen( self, prompt: str, do_sample: bool = True, num_samples: int = 200 @@ -475,6 +481,7 @@ def make_model( tp=1, base_url=None, trust_remote_code=False, + tokenizer_name=None, ): if backend == "vllm": return GeneralVllmDecoder( @@ -484,6 +491,7 @@ def make_model( dataset=dataset, tp=tp, trust_remote_code=trust_remote_code, + tokenizer_name=tokenizer_name, ) elif backend == "hf": return GenenralHfTorchDecoder( @@ -492,6 +500,7 @@ def make_model( temperature=temperature, dataset=dataset, trust_remote_code=trust_remote_code, + tokenizer_name=tokenizer_name, ) elif backend == "openai": return OpenAIChatDecoder( From fa55acc02bd3386598c79ca91859212e4040a317 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 04:42:02 +0800 Subject: [PATCH 073/325] update doc --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 814cc37..c68f729 100755 --- a/README.md +++ b/README.md @@ -117,7 +117,8 @@ bigcodebench.generate \ --backend [vllm|hf|openai|mistral|anthropic|google] \ --tp [gpu_number] \ [--trust_remote_code] \ - [--base_url [base_url]] + [--base_url [base_url]] \ + [--tokenizer_name [tokenizer_name]] ``` > The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: From c4ebd8dc832217ea8ac6f2adb157c5398c476114 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 04:47:30 +0800 Subject: [PATCH 074/325] fix: update greedy logic --- bigcodebench/generate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 6b3a7c2..a739b03 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -123,8 +123,7 @@ def main(): args = parser.parse_args() - if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\ - or (args.temperature == 0 and args.n_samples == 1): + if args.greedy or (args.temperature == 0 and args.n_samples == 1): args.temperature = 0 args.bs = 1 args.n_samples = 1 From 677a130579bb7af56b0ec6acbd6a4c311d401965 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 17:10:12 +0800 Subject: [PATCH 075/325] fix: rm legacy --- bigcodebench/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/model.py b/bigcodebench/model.py index 10fba4a..2b20c4d 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -133,7 +133,7 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: if self.tokenizer_name is None: self.tokenizer_name = self.name - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, legacy=False, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) self.llm = LLM(model=name, max_model_len=2048, **kwargs) @@ -193,7 +193,7 @@ def __init__(self, name: str, dataset: str, **kwargs): if self.tokenizer_name is None: self.tokenizer_name = self.name - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, legacy=False, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) From a57676369c40925ce4348ab8c96ce07b5e37a70d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 17:11:55 +0800 Subject: [PATCH 076/325] fix: change split and subset args --- run.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/run.sh b/run.sh index 4b72f85..f116642 100755 --- a/run.sh +++ b/run.sh @@ -15,25 +15,26 @@ else BASE_MODEL=$MODEL fi -FILE_HEADER=$ORG$BASE_MODEL--$DATASET-$SUBSET--$BACKEND-$TEMP-$N_SAMPLES +if [ "$SUBSET" = "full" ]; then + FILE_HEADER="${ORG}${BASE_MODEL}--${DATASET}-${SPLIT}--${BACKEND}-${TEMP}-${N_SAMPLES}" + else + FILE_HEADER="${ORG}${BASE_MODEL}--${DATASET}-${SUBSET}-${SPLIT}--${BACKEND}-${TEMP}-${N_SAMPLES}" + fi echo $FILE_HEADER bigcodebench.generate \ --tp $NUM_GPU \ --model $MODEL \ - --bs $BS \ - --temperature $TEMP \ - --n_samples $N_SAMPLES \ --resume \ --split $SPLIT \ --subset $SUBSET \ --backend $BACKEND \ - --trust_remote_code + --greedy bigcodebench.sanitize --samples $FILE_HEADER.jsonl --calibrate # Check if the ground truth works on your machine -bigcodebench.evaluate --split $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl +bigcodebench.evaluate --split $SPLIT --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl # If the execution is slow: -bigcodebench.evaluate --split $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl --parallel 32 \ No newline at end of file +bigcodebench.evaluate --split $SPLIT --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl --parallel 32 \ No newline at end of file From a47680de9019ccefc5ba68c698ced78647e0a191 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 17:12:36 +0800 Subject: [PATCH 077/325] add more models --- analysis/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysis/utils.py b/analysis/utils.py index e78d8c2..ba2c1a5 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -640,7 +640,7 @@ }, "deepseek-coder": { "name": "DeepSeek-Coder-V2-Instruct", - "link": "https://www.deepseek.com/", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct", "prompted": True, "moe": True, "size": 236, From 700ecae9d46c8111da92613f12cee5d23a6cf3f0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 17:32:18 +0800 Subject: [PATCH 078/325] feat: add legacy option --- bigcodebench/generate.py | 4 +++- bigcodebench/model.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index a739b03..679300c 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -119,6 +119,7 @@ def main(): parser.add_argument("--base_url", default=None, type=str) parser.add_argument("--tp", default=1, type=int) parser.add_argument("--trust_remote_code", action="store_true") + parser.add_argument("--tokenizer_legacy", action="store_true") parser.add_argument("--tokenizer_name", default=None, type=str) args = parser.parse_args() @@ -144,7 +145,8 @@ def main(): base_url=args.base_url, tp=args.tp, trust_remote_code=args.trust_remote_code, - tokenizer_name=args.tokenizer_name + tokenizer_name=args.tokenizer_name, + tokenizer_legacy=args.tokenizer_legacy ) extra = "-" + args.subset if args.subset != "full" else "" diff --git a/bigcodebench/model.py b/bigcodebench/model.py index 2b20c4d..a00c214 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -92,6 +92,7 @@ def __init__( dtype: str = "bfloat16", # default trust_remote_code: bool = False, tokenizer_name: str = None, + tokenizer_legacy: bool = False, ) -> None: print("Initializing a decoder model: {} ...".format(name)) self.name = name @@ -103,6 +104,7 @@ def __init__( self.dtype = dtype self.trust_remote_code = trust_remote_code self.tokenizer_name = tokenizer_name + self.tokenizer_legacy = tokenizer_legacy @abstractmethod def codegen( @@ -133,7 +135,7 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: if self.tokenizer_name is None: self.tokenizer_name = self.name - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=not self.tokenizer_legacy) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) self.llm = LLM(model=name, max_model_len=2048, **kwargs) @@ -193,7 +195,7 @@ def __init__(self, name: str, dataset: str, **kwargs): if self.tokenizer_name is None: self.tokenizer_name = self.name - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=not self.tokenizer_legacy) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) @@ -249,7 +251,8 @@ def __init__(self, name: str, **kwargs): super().__init__(name=name, **kwargs) self.eos += ["\n```\n"] print(f"EOS strings: {self.eos}") - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name, + **kwargs, legacy=not self.tokenizer_legacy) def codegen( self, prompt: str, do_sample: bool = True, num_samples: int = 200 @@ -483,6 +486,7 @@ def make_model( base_url=None, trust_remote_code=False, tokenizer_name=None, + tokenizer_legacy=True, ): if backend == "vllm": return GeneralVllmDecoder( @@ -493,6 +497,7 @@ def make_model( tp=tp, trust_remote_code=trust_remote_code, tokenizer_name=tokenizer_name, + tokenizer_legacy=tokenizer_legacy, ) elif backend == "hf": return GenenralHfTorchDecoder( @@ -502,6 +507,7 @@ def make_model( dataset=dataset, trust_remote_code=trust_remote_code, tokenizer_name=tokenizer_name, + tokenizer_legacy=tokenizer_legacy, ) elif backend == "openai": return OpenAIChatDecoder( From 02283b0a65debfab3a16c8acf4ffe382532cfb93 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 18:01:07 +0800 Subject: [PATCH 079/325] doc: update issue --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c68f729..49ea632 100755 --- a/README.md +++ b/README.md @@ -347,6 +347,8 @@ We share pre-generated code samples from LLMs we have [evaluated](https://huggin ## 🐞 Known Issues +- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizer may be broken and will degrade the performance of the evaluation. Please try `--tokenizer_legacy` during the generation. + - [ ] Due to the flakes in the evaluation, the execution results may vary slightly (~0.2%) between runs. We are working on improving the evaluation stability. - [ ] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. From c84f95b6cb0c58d1ebfc44852e3d841413f7abf6 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 18:08:33 +0800 Subject: [PATCH 080/325] fix: update the legacy behaviour --- README.md | 2 +- bigcodebench/model.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 49ea632..d70e235 100755 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ We share pre-generated code samples from LLMs we have [evaluated](https://huggin ## 🐞 Known Issues -- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizer may be broken and will degrade the performance of the evaluation. Please try `--tokenizer_legacy` during the generation. +- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizer may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected change, please try `--tokenizer_legacy` during the generation. - [ ] Due to the flakes in the evaluation, the execution results may vary slightly (~0.2%) between runs. We are working on improving the evaluation stability. diff --git a/bigcodebench/model.py b/bigcodebench/model.py index a00c214..7dd77b7 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -135,7 +135,7 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: if self.tokenizer_name is None: self.tokenizer_name = self.name - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=not self.tokenizer_legacy) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=self.tokenizer_legacy) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) self.llm = LLM(model=name, max_model_len=2048, **kwargs) @@ -195,7 +195,7 @@ def __init__(self, name: str, dataset: str, **kwargs): if self.tokenizer_name is None: self.tokenizer_name = self.name - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=not self.tokenizer_legacy) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=self.tokenizer_legacy) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) @@ -252,7 +252,7 @@ def __init__(self, name: str, **kwargs): self.eos += ["\n```\n"] print(f"EOS strings: {self.eos}") self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name, - **kwargs, legacy=not self.tokenizer_legacy) + **kwargs, legacy=self.tokenizer_legacy) def codegen( self, prompt: str, do_sample: bool = True, num_samples: int = 200 From e58413aeeba37ab8537c3558698093977892c64f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 18:15:20 +0800 Subject: [PATCH 081/325] update example --- run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/run.sh b/run.sh index f116642..f33fe01 100755 --- a/run.sh +++ b/run.sh @@ -23,7 +23,6 @@ if [ "$SUBSET" = "full" ]; then echo $FILE_HEADER bigcodebench.generate \ - --tp $NUM_GPU \ --model $MODEL \ --resume \ --split $SPLIT \ From 6f1017383b585b7bda2537161fcf3075f9292ee2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 22:55:36 +0800 Subject: [PATCH 082/325] add reproducible hard set code --- analysis/bcb_subset.py | 169 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 analysis/bcb_subset.py diff --git a/analysis/bcb_subset.py b/analysis/bcb_subset.py new file mode 100644 index 0000000..0c6d8b1 --- /dev/null +++ b/analysis/bcb_subset.py @@ -0,0 +1,169 @@ +import pickle +import json +import numpy as np +from tqdm import tqdm +from ast import literal_eval +from glob import glob +from sentence_transformers import SentenceTransformer, util +import matplotlib.pyplot as plt +from transformers import AutoTokenizer +from datasets import load_dataset, Dataset, Features, Value, Sequence, DatasetDict + +from utils import * + +VERSION = "v0.1.0_hf" +def update_model_info(model_info): + for model, info in model_info.items(): + if "https://huggingface.co/" in info["link"]: + hf_model = info["link"].split("https://huggingface.co/")[-1] + print(hf_model) + tokenizer = AutoTokenizer.from_pretrained(hf_model, trust_remote_code=True) + if tokenizer.chat_template is None: + model_info[model]["direct_complete"] = True + else: + model_info[model]["direct_complete"] = False + else: + model_info[model]["direct_complete"] = False + + return model_info + + +def embed_sentences(data, col_name, id_name, model, save_path, push_to_hub=False): + pool = model.start_multi_process_pool() + embeddings = model.encode_multi_process(data[col_name], pool=pool) + qids = data[id_name] + features = Features({id_name: Value(dtype='string'), 'embeddings': Sequence(Value('float32'))}) + embed_dict = { + id_name: qids, + "embeddings": embeddings + } + embed_ds = Dataset.from_dict(embed_dict, features=features) + if push_to_hub: + embed_ds.push_to_hub(f"bigcode/{save_path}") + else: + embed_ds.save_to_disk(save_path) + return embed_ds + + +def get_top_docs(query_embs, doc_emb, docs): + scores = np.dot(query_embs, doc_emb.T) + top_doc_indices = np.argmax(scores, axis=1) + top_scores = scores[np.arange(len(scores)), top_doc_indices] + results = [(i, docs[doc_idx], score) for i, (doc_idx, score) in tqdm(enumerate(zip(top_doc_indices, top_scores)))] + + return results + + +def filter_top_k_percent(results, k_percent): + all_scores = [score for _, score in results] + threshold = np.percentile(all_scores, 100 - k_percent) + filtered_results = [(i, doc, score) for i, doc, score in results if score > threshold] + return filtered_results + + +def filter_top_threshold(results, threshold): + filtered_results = [(i, doc, score) for i, doc, score in results if score > threshold] + return filtered_results + + +def read_task_perf(tids, task="complete"): + model_results = dict() + result_files = [] + for model, info in model_info.items(): + if task == "instruct" and (not info["prompted"] or info["name"] in ["Granite-Code-3B-Instruct", "Granite-Code-8B-Instruct"]): + continue + task_perf = {f"BigCodeBench/{task_id}": 0 for task_id in range(1140)} + model = model.replace("/", "--") + # if info["link"].startswith("https://huggingface.co/"): + # model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") + try: + if info["prompted"] and not info["direct_complete"]: + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") + if files: + file = files[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + except: + continue + with open(file, "r") as f: + data = json.load(f) + for task_id, perfs in data["eval"].items(): + status = 1 if perfs[0]["status"] == "pass" else 0 + task_perf[task_id] = status + model_results[info["name"]] = np.mean([status for tid, status in task_perf.items() if tid in tids]) + return sorted(model_results.items(), key=lambda x: x[1], reverse=True) + + +if __name__ == "__main__": + bcb = load_dataset("bigcode/bigcodebench", trust_remote_code=True, split=VERSION) + se = load_dataset("bigcode/stack-exchange-preferences-20230914-clean-anonymization", trust_remote_code=True, split="train") + model = SentenceTransformer("sentence-transformers/all-mpnet-base-v2") + + model_info = update_model_info(model_info) + + se_embed = embed_sentences(se, "question", "qid", model, "stack-exchange-embeddings-20230914", push_to_hub=True) + bcb_embed = embed_sentences(bcb, "complete_prompt", "task_id", model, "bigcodebench-doc-embeddings", push_to_hub=True) + + solve_rate = load_dataset("bigcode/bigcodebench-solve-rate", trust_remote_code=True, split="complete") + + query_embs = np.array(se_embed["embeddings"]) + doc_emb = np.array(bcb_embed["embeddings"]) + docs = bcb_embed["task_id"] + retrieval_results = get_top_docs(query_embs, doc_emb, docs) + + Dataset.from_dict({"qid": [i for i, _, _ in retrieval_results], "tid": [doc for _, doc, _ in retrieval_results], "score": [score for _, _, score in retrieval_results]}).push_to_hub("bigcode/se_bcb_results") + + retrieval_ds = load_dataset("bigcode/se_bcb_results", trust_remote_code=True, split="train") + + top_results = dict() + for sample in tqdm(retrieval_ds): + i, doc, score = sample["qid"], sample["tid"], sample["score"] + if score > 0.7: + if doc not in top_results: + top_results[doc] = (i, doc, score) + else: + if score > top_results[doc][2]: + top_results[doc] = (i, doc, score) + + top_id = {task_id: (qid, score) for qid, task_id, score in top_results.values()} + + hard_lib_filter = {sample["task_id"] for sample in bcb if len(literal_eval(sample["libs"])) > 2} + hard_length_filter = {sample["task_id"] for sample in bcb if len(sample["canonical_solution"]) > 426} + hard_rate_filter = {task["task_id"]: task["solve_rate"] for task in solve_rate if task["solve_rate"] < 50} + + hard_tid = top_id.keys() & hard_length_filter & hard_rate_filter.keys() & hard_lib_filter + + hard_bcb = bcb.filter(lambda x: x["task_id"] in hard_tid) + hard_bcb_tid = bcb.filter(lambda x: x["task_id"] in hard_tid)["task_id"] + hard_se_qid = [top_id[_id][0] for _id in hard_bcb_tid] + hard_se_q = se.select(hard_se_qid) + hard_se_scores = [top_id[_id][1] for _id in hard_bcb_tid] + hard_bcb_dict = { + "task_id": hard_bcb_tid, + "complete_prompt": hard_bcb["complete_prompt"], + "instruct_prompt": hard_bcb["instruct_prompt"], + "canonical_solution": hard_bcb["canonical_solution"], + "code_prompt": hard_bcb["code_prompt"], + "test": hard_bcb["test"], + "entry_point": hard_bcb["entry_point"], + "doc_struct": hard_bcb["doc_struct"], + "libs": hard_bcb["libs"], + "q_idx": hard_se_qid, + "question": hard_se_q["question"], + "score": hard_se_scores, + "_id": hard_bcb_tid + } + hard_bcb = Dataset.from_dict(hard_bcb_dict) + DatasetDict({VERSION: hard_bcb}).push_to_hub("bigcode/bigcodebench-hard") + + hard_complete_results = read_task_perf(hard_tid) + hard_instruct_results = read_task_perf(hard_tid, task="instruct") + + complete_res_dict = {model: score for model, score in hard_complete_results} + instruct_res_dict = {model: score for model, score in hard_instruct_results} + avg_res_dict = {model: (complete_res_dict[model] + instruct_res_dict[model]) / 2 for model in complete_res_dict if model in instruct_res_dict} + + for model, score in sorted(avg_res_dict.items(), key=lambda x: x[1], reverse=True): + print(model, round(score*100, 1)) \ No newline at end of file From add46519c37bca10a0803da36f1c5f701c65d359 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 23:23:07 +0800 Subject: [PATCH 083/325] fix: update pass_k_res --- bigcodebench/evaluate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index f7fc31f..8cc91d8 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -283,7 +283,8 @@ def stucking_checker(): json.dump(results, f, indent=2) pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") - pass_at_k["model"] = flags.samples.split("/")[-1].replace(".jsonl", "") + pass_at_k["model"] = os.path.basename(flags.samples).split("--bigcodebench-")[0] + pass_at_k["calibrated"] = "sanitized-calibrated" in flags.samples pass_at_k["subset"] = flags.subset def save_pass_at_k(): From aa10cd1dc34097b78bfe9559ce872a6df7fb8195 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 23:40:33 +0800 Subject: [PATCH 084/325] fix typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d70e235..58f77ec 100755 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ We share pre-generated code samples from LLMs we have [evaluated](https://huggin ## 🐞 Known Issues -- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizer may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected change, please try `--tokenizer_legacy` during the generation. +- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizer may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. - [ ] Due to the flakes in the evaluation, the execution results may vary slightly (~0.2%) between runs. We are working on improving the evaluation stability. From 32f5382d1a677eb57ae2508d3abe2e8a47e15fea Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 17 Jul 2024 23:41:23 +0800 Subject: [PATCH 085/325] fix: remove commented lines --- analysis/bcb_subset.py | 2 -- analysis/get_results.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/analysis/bcb_subset.py b/analysis/bcb_subset.py index 0c6d8b1..7760a92 100644 --- a/analysis/bcb_subset.py +++ b/analysis/bcb_subset.py @@ -74,8 +74,6 @@ def read_task_perf(tids, task="complete"): continue task_perf = {f"BigCodeBench/{task_id}": 0 for task_id in range(1140)} model = model.replace("/", "--") - # if info["link"].startswith("https://huggingface.co/"): - # model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") try: if info["prompted"] and not info["direct_complete"]: files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") diff --git a/analysis/get_results.py b/analysis/get_results.py index 196ba26..83af2b3 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -52,9 +52,6 @@ def get_results(tids): hf_model = "" files = glob(f"results/{model}--bigcodebench-*.json") assert files, f"No files found for results/{model}--bigcodebench-*.json" - # if "https://huggingface.co/" in info["link"]: - # hf_model = info["link"].split("https://huggingface.co/")[-1] - # model = hf_model.replace("/", "--") for file in files: _, suffix = os.path.basename(file).split("--bigcodebench-") status = [] @@ -153,8 +150,6 @@ def read_task_perf(tids, task="complete"): task_perf = dict() model = model.replace("/", "--") - # if info["link"].startswith("https://huggingface.co/"): - # model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") try: if info["prompted"] and not info["direct_complete"]: files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") From 164d6e466a1de93ded5b46b8ceb1d820520e5627 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 18 Jul 2024 01:57:17 +0800 Subject: [PATCH 086/325] add hard version --- analysis/bcb_subset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/analysis/bcb_subset.py b/analysis/bcb_subset.py index 0c6d8b1..b211efa 100644 --- a/analysis/bcb_subset.py +++ b/analysis/bcb_subset.py @@ -12,6 +12,7 @@ from utils import * VERSION = "v0.1.0_hf" + def update_model_info(model_info): for model, info in model_info.items(): if "https://huggingface.co/" in info["link"]: From f1b10cf3d6ea56f636dfe37c5b202dd5d6e433fb Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 18 Jul 2024 02:02:03 +0800 Subject: [PATCH 087/325] update models --- analysis/utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/analysis/utils.py b/analysis/utils.py index ba2c1a5..4b1307c 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -602,7 +602,7 @@ "act_param": 7, "open-data": "None", }, - "mistralai/Codestral-22B-v0.1": { + "codestral-2405": { "name": "Codestral-22B-v0.1", "link": "https://huggingface.co/mistralai/Codestral-22B-v0.1", "prompted": True, @@ -611,6 +611,15 @@ "act_param": 22, "open-data": "None", }, + "codestral-mamba-2407": { + "name": "Codestral-Mamba", + "link": "https://huggingface.co/mistralai/mamba-codestral-7B-v0.1", + "prompted": True, + "moe": False, + "size": 7, + "act_param": 7, + "open-data": "None", + }, "mistralai/Mistral-7B-Instruct-v0.3": { "name": "Mistral-7B-Instruct-v0.3", "link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3", From c65100d1fc13759dc9a55fd784181c8738af579a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 19 Jul 2024 05:35:42 +0800 Subject: [PATCH 088/325] doc: fix typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 58f77ec..cf40c4d 100755 --- a/README.md +++ b/README.md @@ -347,9 +347,9 @@ We share pre-generated code samples from LLMs we have [evaluated](https://huggin ## 🐞 Known Issues -- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizer may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. +- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizers may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. -- [ ] Due to the flakes in the evaluation, the execution results may vary slightly (~0.2%) between runs. We are working on improving the evaluation stability. +- [ ] Due to the flakiness in the evaluation, the execution results may vary slightly (~0.2% for Full set, and ~0.6% for Hard set) between runs. We are working on improving the evaluation stability. - [ ] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. From 057a84a4b576da4ba2c856aa9bbbb80211f1d0f8 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 21 Jul 2024 23:14:40 +0800 Subject: [PATCH 089/325] feat: make pass_rate_save optional --- bigcodebench/evaluate.py | 52 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 8cc91d8..b849423 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -281,33 +281,34 @@ def stucking_checker(): if not os.path.isfile(result_path): with open(result_path, "w") as f: json.dump(results, f, indent=2) - - pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") - pass_at_k["model"] = os.path.basename(flags.samples).split("--bigcodebench-")[0] - pass_at_k["calibrated"] = "sanitized-calibrated" in flags.samples - pass_at_k["subset"] = flags.subset - - def save_pass_at_k(): - with open(pass_at_k_path, "w") as f: - json.dump(pass_at_k, f, indent=2) - - if os.path.isfile(pass_at_k_path): - saved_pass_at_k = json.load(open(pass_at_k_path, "r")) - # compare saved_pass_at_k with pass_at_k - for k in saved_pass_at_k.keys(): - if pass_at_k[k] != saved_pass_at_k[k]: - cprint(f"Warning: {k} is different from the saved one", "yellow") + + if flags.save_pass_rate: + pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") + pass_at_k["model"] = os.path.basename(flags.samples).split("--bigcodebench-")[0] + pass_at_k["calibrated"] = "sanitized-calibrated" in flags.samples + pass_at_k["subset"] = flags.subset + + def save_pass_at_k(): + with open(pass_at_k_path, "w") as f: + json.dump(pass_at_k, f, indent=2) + + if os.path.isfile(pass_at_k_path): + saved_pass_at_k = json.load(open(pass_at_k_path, "r")) + # compare saved_pass_at_k with pass_at_k + for k in saved_pass_at_k.keys(): + if pass_at_k[k] != saved_pass_at_k[k]: + cprint(f"Warning: {k} is different from the saved one", "yellow") + + # ask user whether to save the pass@k + decision = "" + while decision.lower() not in ["y", "n"]: + print(f"Save pass@k to {pass_at_k_path}? [Y/N]") + decision = input() + if decision.lower() == "y": + save_pass_at_k() - # ask user whether to save the pass@k - decision = "" - while decision.lower() not in ["y", "n"]: - print(f"Save pass@k to {pass_at_k_path}? [Y/N]") - decision = input() - if decision.lower() == "y": + else: save_pass_at_k() - - else: - save_pass_at_k() def main(): @@ -317,6 +318,7 @@ def main(): ) parser.add_argument("--subset", default="full", type=str, choices=["full", "hard"]) parser.add_argument("--samples", required=True, type=str) + parser.add_argument("--save_pass_rate", action="store_true") parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) parser.add_argument("--max-as-limit", default=128*1024, type=int) From 7561435bf86a5bfdaf4458765c68121d86b5edc6 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 23 Jul 2024 01:21:44 +0800 Subject: [PATCH 090/325] fix: update help eval info --- bigcodebench/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index b849423..6ca7215 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -325,10 +325,10 @@ def main(): parser.add_argument("--max-data-limit", default=4*1024, type=int) parser.add_argument("--max-stack-limit", default=5, type=int) parser.add_argument( - "--check-gt-only", action="store_true", help="Check the groundtruth" + "--check-gt-only", action="store_true", help="Check the ground truth" ) parser.add_argument( - "--no-gt", action="store_true", help="Check the groundtruth" + "--no-gt", action="store_true", help="Skip the ground truth" ) args = parser.parse_args() From bbaf0cc99c0701add5d1d3b4abdda7c519903f16 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 23 Jul 2024 21:06:57 +0800 Subject: [PATCH 091/325] feat: add failed tasks --- bigcodebench/evaluate.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 6ca7215..76d8e23 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -134,6 +134,7 @@ def evaluate(flags): expected_time = {task_id: None for task_id in problems} gt_pass_rate = np.mean([1 if v is not None else 0 for k, v in expected_time.items() if k in problems]) + failed_tasks = [k for k, v in expected_time.items() if v is None and k in problems] if os.path.isfile(result_path): print(f"Load from previous results from {result_path}") @@ -150,6 +151,9 @@ def evaluate(flags): cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") return + if len(failed_tasks) > 0: + cprint(f"Failed tasks: {failed_tasks}", "red") + results = { "date": datetime.now().strftime("%Y-%m-%d %H:%M"), "eval": {}, @@ -259,6 +263,9 @@ def stucking_checker(): cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") else: cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") + + if len(failed_tasks) > 0: + cprint(f"Failed tasks: {failed_tasks}", "red") for k, v in pass_at_k.items(): cprint(f"{k}:\t{v:.3f}", "green") From 985b1086e05d0c0d6c557fbc68dcaeb77a542563 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 23 Jul 2024 21:09:56 +0800 Subject: [PATCH 092/325] feat: add gradio docker --- Docker/Gradio.Dockerfile | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 Docker/Gradio.Dockerfile diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile new file mode 100644 index 0000000..9d56157 --- /dev/null +++ b/Docker/Gradio.Dockerfile @@ -0,0 +1,37 @@ +# Better use newer Python as generated code can use new features +FROM python:3.10-slim + +# install git, g++ and python3-tk +RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-base + +# upgrade to latest pip +RUN pip install --upgrade pip + +RUN pip install gradio==4.31.0 gradio[oauth] +# Add a new user "bigcodebenchuser" +RUN adduser --disabled-password --gecos "" bigcodebenchuser + +RUN rm -rf /bigcodebench + +# Acquire benchmark code to local +ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench + +RUN cd /bigcodebench && pip install . + +# Pre-install the dataset +RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" + +RUN pip install -I --timeout 2000 -r https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0/requirements.txt + +WORKDIR /app + +RUN chown -R bigcodebenchuser:bigcodebenchuser /app + +RUN chmod -R 777 /app + +USER bigcodebenchuser + +# ENTRYPOINT ["python", "app.py"] + +# CMD ["sh", "-c", "pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/*"] \ No newline at end of file From 2a11c993392603adafaafb370b47c242b9c31f73 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 25 Jul 2024 15:18:54 +0800 Subject: [PATCH 093/325] add domain details --- analysis/lib2domain.json | 157 ++ analysis/task2domain.json | 5433 +++++++++++++++++++++++++++++++++++++ 2 files changed, 5590 insertions(+) create mode 100644 analysis/lib2domain.json create mode 100644 analysis/task2domain.json diff --git a/analysis/lib2domain.json b/analysis/lib2domain.json new file mode 100644 index 0000000..d06a48a --- /dev/null +++ b/analysis/lib2domain.json @@ -0,0 +1,157 @@ +{ + "Crypto": "Cryptography", + "PIL": "Visualization", + "array": "General", + "base64": "Cryptography", + "binascii": "Cryptography", + "bisect": "General", + "blake3": "Cryptography", + "bs4": "Network", + "calendar": "Time", + "cgi": "Network", + "chardet": "Network", + "cmath": "Computation", + "codecs": "Cryptography", + "collections": "General", + "cryptography": "Cryptography", + "csv": "System", + "ctypes": "System", + "datetime": "Time", + "dateutil": "Time", + "difflib": "General", + "django": "Network", + "docx": "System", + "email": "Network", + "faker": "General", + "flask": "Network", + "flask_login": "Network", + "flask_mail": "Network", + "flask_restful": "Network", + "fnmatch": "General", + "folium": "Visualization", + "functools": "General", + "geopy": "Network", + "getpass": "System", + "glob": "System", + "gzip": "System", + "hashlib": "Cryptography", + "heapq": "General", + "hmac": "Cryptography", + "html": "Network", + "http": "Network", + "importlib": "General", + "inspect": "General", + "io": "System", + "ipaddress": "Network", + "itertools": "General", + "json": "System", + "keras": "Computation", + "librosa": "Computation", + "logging": "System", + "lxml": "Network", + "math": "Computation", + "matplotlib": "Visualization", + "mechanize": "Network", + "mimetypes": "Network", + "multiprocessing": "System", + "nltk": "Computation", + "numpy": "Computation", + "openpyxl": "System", + "operator": "General", + "os": "System", + "pandas": "Computation", + "pathlib": "System", + "pickle": "System", + "pkgutil": "General", + "platform": "System", + "prettytable": "General", + "psutil": "System", + "pytesseract": "Computation", + "pytz": "Time", + "queue": "General", + "random": "General", + "re": "General", + "requests": "Network", + "rsa": "Cryptography", + "scipy": "Computation", + "seaborn": "Visualization", + "secrets": "Cryptography", + "select": "System", + "sendgrid": "Network", + "shutil": "System", + "sklearn": "Computation", + "smtplib": "Network", + "socket": "Network", + "soundfile": "Computation", + "sqlite3": "System", + "ssl": "Network", + "statistics": "Computation", + "statsmodels": "Computation", + "string": "General", + "struct": "System", + "subprocess": "System", + "sys": "System", + "tarfile": "System", + "tensorflow": "Computation", + "texttable": "General", + "textwrap": "General", + "threading": "System", + "time": "Time", + "turtle": "Visualization", + "types": "General", + "unicodedata": "General", + "urllib": "Network", + "uuid": "General", + "warnings": "General", + "werkzeug": "Network", + "wordninja": "Computation", + "wtforms": "Network", + "xlwt": "System", + "xml": "Network", + "xmltodict": "Network", + "yaml": "System", + "zipfile": "System", + "Levenshtein": "Computation", + "ast": "General", + "configparser": "System", + "cv2": "Computation", + "decimal": "General", + "enum": "General", + "errno": "System", + "flask_wtf": "Network", + "ftplib": "Network", + "gensim": "Computation", + "geopandas": "Computation", + "holidays": "Time", + "mpl_toolkits": "Visualization", + "natsort": "General", + "pyquery": "Network", + "python_http_client": "Network", + "regex": "General", + "shapely": "Computation", + "shlex": "System", + "signal": "System", + "skimage": "Computation", + "sympy": "Computation", + "textblob": "Computation", + "typing": "General", + "wikipedia": "Network", + "wordcloud": "Visualization", + "zlib": "System", + "aspose": "System", + "builtins": "General", + "locale": "System", + "imp": "System", + "docxtpl": "System", + "selenium": "Network", + "IPython": "Computation", + "filecmp": "System", + "multidict": "General", + "sqlalchemy": "System", + "obspy": "Computation", + "pprint": "General", + "xlrd": "System", + "argparse": "General", + "torch": "Computation", + "copy": "General" +} \ No newline at end of file diff --git a/analysis/task2domain.json b/analysis/task2domain.json new file mode 100644 index 0000000..fefa259 --- /dev/null +++ b/analysis/task2domain.json @@ -0,0 +1,5433 @@ +{ + "BigCodeBench/0": [ + "General", + "General" + ], + "BigCodeBench/1": [ + "General", + "General", + "General" + ], + "BigCodeBench/2": [ + "Computation", + "General" + ], + "BigCodeBench/3": [ + "Computation", + "General" + ], + "BigCodeBench/4": [ + "General", + "General" + ], + "BigCodeBench/5": [ + "Computation", + "General" + ], + "BigCodeBench/6": [ + "General", + "System" + ], + "BigCodeBench/7": [ + "General", + "System", + "General" + ], + "BigCodeBench/8": [ + "General", + "General", + "General" + ], + "BigCodeBench/9": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/10": [ + "Computation", + "Computation", + "General", + "General" + ], + "BigCodeBench/11": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/12": [ + "System", + "Time", + "System", + "System" + ], + "BigCodeBench/13": [ + "System", + "Network", + "System" + ], + "BigCodeBench/14": [ + "System", + "System", + "System" + ], + "BigCodeBench/15": [ + "System", + "System", + "System" + ], + "BigCodeBench/16": [ + "System", + "System", + "System" + ], + "BigCodeBench/17": [ + "System", + "System", + "Time" + ], + "BigCodeBench/18": [ + "System", + "System", + "General", + "System", + "System" + ], + "BigCodeBench/19": [ + "System", + "System", + "System" + ], + "BigCodeBench/20": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/21": [ + "System", + "System" + ], + "BigCodeBench/22": [ + "General", + "General", + "General" + ], + "BigCodeBench/23": [ + "Computation", + "General" + ], + "BigCodeBench/24": [ + "Cryptography", + "Cryptography", + "System" + ], + "BigCodeBench/25": [ + "System", + "Cryptography", + "System" + ], + "BigCodeBench/26": [ + "Cryptography", + "Cryptography" + ], + "BigCodeBench/27": [ + "Cryptography", + "System", + "Time" + ], + "BigCodeBench/28": [ + "Cryptography", + "Network", + "System" + ], + "BigCodeBench/29": [ + "Cryptography", + "Computation", + "Computation" + ], + "BigCodeBench/30": [ + "System", + "General", + "System" + ], + "BigCodeBench/31": [ + "Computation", + "Visualization", + "General", + "Visualization" + ], + "BigCodeBench/32": [ + "Network", + "Network" + ], + "BigCodeBench/33": [ + "Computation", + "General" + ], + "BigCodeBench/34": [ + "Visualization", + "General", + "Visualization" + ], + "BigCodeBench/35": [ + "Visualization", + "Visualization" + ], + "BigCodeBench/36": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/37": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/38": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/39": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/40": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/41": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/42": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/43": [ + "Computation", + "Visualization" + ], + "BigCodeBench/44": [ + "Visualization", + "Computation" + ], + "BigCodeBench/45": [ + "Computation", + "Visualization", + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/46": [ + "Visualization", + "Computation" + ], + "BigCodeBench/47": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/48": [ + "Time", + "General", + "Visualization", + "Time" + ], + "BigCodeBench/49": [ + "Computation", + "Time", + "Visualization" + ], + "BigCodeBench/50": [ + "Time", + "Computation", + "Time", + "Visualization" + ], + "BigCodeBench/51": [ + "Visualization", + "Computation" + ], + "BigCodeBench/52": [ + "General", + "Computation" + ], + "BigCodeBench/53": [ + "General", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/54": [ + "General", + "Computation", + "Computation" + ], + "BigCodeBench/55": [ + "Computation", + "General" + ], + "BigCodeBench/56": [ + "General", + "Computation" + ], + "BigCodeBench/57": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/58": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/59": [ + "Network", + "Visualization", + "Visualization" + ], + "BigCodeBench/60": [ + "Computation", + "System" + ], + "BigCodeBench/61": [ + "Time", + "Computation", + "Visualization" + ], + "BigCodeBench/62": [ + "General", + "Visualization", + "Visualization" + ], + "BigCodeBench/63": [ + "Computation", + "Visualization" + ], + "BigCodeBench/64": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/65": [ + "Computation", + "Visualization" + ], + "BigCodeBench/66": [ + "Computation", + "Visualization" + ], + "BigCodeBench/67": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/68": [ + "Computation", + "Visualization" + ], + "BigCodeBench/69": [ + "General", + "Visualization" + ], + "BigCodeBench/70": [ + "Computation", + "Computation", + "System" + ], + "BigCodeBench/71": [ + "General", + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/72": [ + "Computation", + "Computation", + "General", + "System" + ], + "BigCodeBench/73": [ + "General", + "Computation", + "Visualization", + "Computation", + "System" + ], + "BigCodeBench/74": [ + "Network", + "Network" + ], + "BigCodeBench/75": [ + "Computation", + "General", + "Computation", + "Visualization", + "Time" + ], + "BigCodeBench/76": [ + "Network", + "General", + "General" + ], + "BigCodeBench/77": [ + "Cryptography", + "Network", + "Cryptography", + "Cryptography" + ], + "BigCodeBench/78": [ + "System", + "System", + "Network" + ], + "BigCodeBench/79": [ + "System", + "Network", + "System" + ], + "BigCodeBench/80": [ + "System", + "Network", + "System" + ], + "BigCodeBench/81": [ + "Network", + "Network", + "Network" + ], + "BigCodeBench/82": [ + "Network", + "Network", + "Network", + "Network", + "Network" + ], + "BigCodeBench/83": [ + "Network", + "Network" + ], + "BigCodeBench/84": [ + "Computation", + "Computation" + ], + "BigCodeBench/85": [ + "Computation", + "Time", + "Computation" + ], + "BigCodeBench/86": [ + "Computation", + "Computation" + ], + "BigCodeBench/87": [ + "Computation", + "General" + ], + "BigCodeBench/88": [ + "Computation", + "Time", + "Computation" + ], + "BigCodeBench/89": [ + "Computation", + "Visualization", + "Computation", + "Computation" + ], + "BigCodeBench/90": [ + "Computation", + "Computation" + ], + "BigCodeBench/91": [ + "Visualization", + "Computation" + ], + "BigCodeBench/92": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/93": [ + "Computation", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/94": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/95": [ + "Computation", + "General" + ], + "BigCodeBench/96": [ + "General", + "System", + "General" + ], + "BigCodeBench/97": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/98": [ + "General", + "General", + "General" + ], + "BigCodeBench/99": [ + "Computation", + "Visualization", + "Visualization", + "Computation" + ], + "BigCodeBench/100": [ + "Computation", + "Time", + "General", + "Visualization" + ], + "BigCodeBench/101": [ + "Computation", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/102": [ + "Computation", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/103": [ + "Computation", + "Visualization" + ], + "BigCodeBench/104": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/105": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/106": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/107": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/108": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/109": [ + "Computation", + "Visualization" + ], + "BigCodeBench/110": [ + "Computation", + "Visualization" + ], + "BigCodeBench/111": [ + "Computation", + "Visualization" + ], + "BigCodeBench/112": [ + "Computation", + "Visualization" + ], + "BigCodeBench/113": [ + "General", + "General", + "System" + ], + "BigCodeBench/114": [ + "Computation", + "Computation" + ], + "BigCodeBench/115": [ + "Computation", + "Computation" + ], + "BigCodeBench/116": [ + "Computation", + "Visualization" + ], + "BigCodeBench/117": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/118": [ + "System", + "System" + ], + "BigCodeBench/119": [ + "Computation", + "Visualization" + ], + "BigCodeBench/120": [ + "Computation", + "Time", + "General" + ], + "BigCodeBench/121": [ + "Computation", + "Computation" + ], + "BigCodeBench/122": [ + "Computation", + "General" + ], + "BigCodeBench/123": [ + "System", + "Computation", + "System" + ], + "BigCodeBench/124": [ + "General", + "Visualization", + "Time" + ], + "BigCodeBench/125": [ + "General", + "General", + "General", + "System" + ], + "BigCodeBench/126": [ + "Computation", + "Computation", + "Computation", + "General" + ], + "BigCodeBench/127": [ + "System", + "System", + "Cryptography", + "System" + ], + "BigCodeBench/128": [ + "Computation", + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/129": [ + "Computation", + "Network", + "Network" + ], + "BigCodeBench/130": [ + "Cryptography", + "Cryptography", + "System", + "Cryptography" + ], + "BigCodeBench/131": [ + "Cryptography", + "Cryptography", + "System", + "Cryptography" + ], + "BigCodeBench/132": [ + "Computation", + "Computation", + "Visualization", + "Cryptography" + ], + "BigCodeBench/133": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/134": [ + "Computation", + "Visualization" + ], + "BigCodeBench/135": [ + "Computation", + "Visualization", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/136": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/137": [ + "Computation", + "Computation" + ], + "BigCodeBench/138": [ + "Computation", + "Visualization" + ], + "BigCodeBench/139": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/140": [ + "Computation", + "Computation" + ], + "BigCodeBench/141": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/142": [ + "Computation", + "Visualization" + ], + "BigCodeBench/143": [ + "Computation", + "Visualization" + ], + "BigCodeBench/144": [ + "Network", + "Network" + ], + "BigCodeBench/145": [ + "System", + "Network" + ], + "BigCodeBench/146": [ + "System", + "Network" + ], + "BigCodeBench/147": [ + "System", + "Network", + "Network" + ], + "BigCodeBench/148": [ + "Computation", + "Computation" + ], + "BigCodeBench/149": [ + "Computation", + "Computation" + ], + "BigCodeBench/150": [ + "Computation", + "Computation" + ], + "BigCodeBench/151": [ + "Computation", + "Computation" + ], + "BigCodeBench/152": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/153": [ + "Computation", + "Computation" + ], + "BigCodeBench/154": [ + "System", + "Network", + "General", + "System" + ], + "BigCodeBench/155": [ + "Computation", + "Visualization" + ], + "BigCodeBench/156": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/157": [ + "Computation", + "Visualization" + ], + "BigCodeBench/158": [ + "Network", + "System", + "System" + ], + "BigCodeBench/159": [ + "System", + "System", + "System" + ], + "BigCodeBench/160": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/161": [ + "Computation", + "Time", + "General" + ], + "BigCodeBench/162": [ + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/163": [ + "Computation", + "Computation" + ], + "BigCodeBench/164": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/165": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/166": [ + "Time", + "Computation", + "Time" + ], + "BigCodeBench/167": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/168": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/169": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/170": [ + "Computation", + "System", + "Network" + ], + "BigCodeBench/171": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/172": [ + "Time", + "System" + ], + "BigCodeBench/173": [ + "Computation", + "Computation" + ], + "BigCodeBench/174": [ + "Computation", + "Computation" + ], + "BigCodeBench/175": [ + "Visualization", + "General" + ], + "BigCodeBench/176": [ + "General", + "Network" + ], + "BigCodeBench/177": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/178": [ + "General", + "System" + ], + "BigCodeBench/179": [ + "Computation", + "Visualization", + "General", + "Computation" + ], + "BigCodeBench/180": [ + "Visualization", + "Visualization", + "Computation", + "System", + "Computation" + ], + "BigCodeBench/181": [ + "Network", + "General", + "Time" + ], + "BigCodeBench/182": [ + "General", + "Computation" + ], + "BigCodeBench/183": [ + "General", + "Network" + ], + "BigCodeBench/184": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/185": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/186": [ + "Network", + "Visualization" + ], + "BigCodeBench/187": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/188": [ + "Computation", + "Network", + "Visualization" + ], + "BigCodeBench/189": [ + "General", + "Network", + "System" + ], + "BigCodeBench/190": [ + "System", + "Computation", + "System", + "System" + ], + "BigCodeBench/191": [ + "General", + "Computation" + ], + "BigCodeBench/192": [ + "Network", + "General" + ], + "BigCodeBench/193": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/194": [ + "Computation", + "Visualization" + ], + "BigCodeBench/195": [ + "System", + "System", + "Time" + ], + "BigCodeBench/196": [ + "Computation", + "Visualization", + "General", + "Visualization" + ], + "BigCodeBench/197": [ + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/198": [ + "Computation", + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/199": [ + "Time", + "Computation", + "Time", + "General" + ], + "BigCodeBench/200": [ + "Computation", + "General", + "General", + "Visualization" + ], + "BigCodeBench/201": [ + "Computation", + "General" + ], + "BigCodeBench/202": [ + "General", + "General", + "System" + ], + "BigCodeBench/203": [ + "Network", + "System" + ], + "BigCodeBench/204": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/205": [ + "System", + "System" + ], + "BigCodeBench/206": [ + "System", + "System", + "System" + ], + "BigCodeBench/207": [ + "General", + "Network" + ], + "BigCodeBench/208": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/209": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/210": [ + "General", + "General", + "Visualization" + ], + "BigCodeBench/211": [ + "System", + "Network", + "System" + ], + "BigCodeBench/212": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/213": [ + "General", + "Visualization", + "Computation", + "Time" + ], + "BigCodeBench/214": [ + "Computation", + "Visualization", + "General", + "Computation" + ], + "BigCodeBench/215": [ + "Computation", + "System", + "Network", + "Visualization" + ], + "BigCodeBench/216": [ + "Computation", + "General", + "System", + "System" + ], + "BigCodeBench/217": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/218": [ + "Computation", + "Computation" + ], + "BigCodeBench/219": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/220": [ + "Visualization", + "General", + "Time" + ], + "BigCodeBench/221": [ + "Computation", + "Computation" + ], + "BigCodeBench/222": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/223": [ + "Computation", + "Computation" + ], + "BigCodeBench/224": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/225": [ + "Computation", + "Visualization" + ], + "BigCodeBench/226": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/227": [ + "Computation", + "Visualization", + "Computation", + "Computation", + "System" + ], + "BigCodeBench/228": [ + "Computation", + "Computation" + ], + "BigCodeBench/229": [ + "Time", + "General", + "System" + ], + "BigCodeBench/230": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/231": [ + "General", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/232": [ + "Computation", + "General" + ], + "BigCodeBench/233": [ + "General", + "Visualization" + ], + "BigCodeBench/234": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/235": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/236": [ + "Computation", + "Computation" + ], + "BigCodeBench/237": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/238": [ + "Visualization", + "Computation" + ], + "BigCodeBench/239": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/240": [ + "Computation", + "General" + ], + "BigCodeBench/241": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/242": [ + "Visualization", + "Computation" + ], + "BigCodeBench/243": [ + "Computation", + "General" + ], + "BigCodeBench/244": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/245": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/246": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/247": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/248": [ + "Visualization", + "Computation", + "General" + ], + "BigCodeBench/249": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/250": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/251": [ + "Computation", + "Visualization" + ], + "BigCodeBench/252": [ + "Visualization", + "General" + ], + "BigCodeBench/253": [ + "Computation", + "General" + ], + "BigCodeBench/254": [ + "Computation", + "System" + ], + "BigCodeBench/255": [ + "Computation", + "Visualization" + ], + "BigCodeBench/256": [ + "Time", + "General", + "Cryptography", + "System" + ], + "BigCodeBench/257": [ + "Computation", + "Computation" + ], + "BigCodeBench/258": [ + "General", + "System" + ], + "BigCodeBench/259": [ + "Computation", + "Visualization" + ], + "BigCodeBench/260": [ + "System", + "System", + "System" + ], + "BigCodeBench/261": [ + "Computation", + "Visualization" + ], + "BigCodeBench/262": [ + "General", + "Visualization", + "Visualization" + ], + "BigCodeBench/263": [ + "System", + "System", + "System", + "Time" + ], + "BigCodeBench/264": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/265": [ + "System", + "General", + "System" + ], + "BigCodeBench/266": [ + "System", + "General", + "System" + ], + "BigCodeBench/267": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/268": [ + "General", + "General" + ], + "BigCodeBench/269": [ + "Computation", + "Visualization", + "Computation", + "Computation" + ], + "BigCodeBench/270": [ + "General", + "General" + ], + "BigCodeBench/271": [ + "Cryptography", + "General", + "General", + "Time" + ], + "BigCodeBench/272": [ + "Network", + "Network", + "System" + ], + "BigCodeBench/273": [ + "Network", + "Network", + "System" + ], + "BigCodeBench/274": [ + "Network", + "Network", + "Network", + "System", + "Network" + ], + "BigCodeBench/275": [ + "Computation", + "General" + ], + "BigCodeBench/276": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/277": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/278": [ + "Computation", + "Computation" + ], + "BigCodeBench/279": [ + "General", + "General" + ], + "BigCodeBench/280": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/281": [ + "General", + "General", + "System" + ], + "BigCodeBench/282": [ + "Visualization", + "Visualization", + "Computation", + "Computation", + "System" + ], + "BigCodeBench/283": [ + "System", + "General", + "System" + ], + "BigCodeBench/284": [ + "Network", + "Network", + "Network" + ], + "BigCodeBench/285": [ + "Network", + "Network" + ], + "BigCodeBench/286": [ + "System", + "General", + "System" + ], + "BigCodeBench/287": [ + "System", + "General", + "System" + ], + "BigCodeBench/288": [ + "System", + "General", + "System" + ], + "BigCodeBench/289": [ + "Computation", + "Computation" + ], + "BigCodeBench/290": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/291": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/292": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/293": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/294": [ + "Computation", + "Computation" + ], + "BigCodeBench/295": [ + "Computation", + "General" + ], + "BigCodeBench/296": [ + "Computation", + "Visualization" + ], + "BigCodeBench/297": [ + "General", + "General" + ], + "BigCodeBench/298": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/299": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/300": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/301": [ + "Time", + "Computation", + "Computation", + "Time" + ], + "BigCodeBench/302": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/303": [ + "Time", + "Computation", + "Computation", + "Time" + ], + "BigCodeBench/304": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/305": [ + "General", + "General", + "General" + ], + "BigCodeBench/306": [ + "System", + "System" + ], + "BigCodeBench/307": [ + "General", + "Visualization", + "Visualization" + ], + "BigCodeBench/308": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/309": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/310": [ + "Computation", + "System", + "General", + "System" + ], + "BigCodeBench/311": [ + "General", + "Computation", + "Computation" + ], + "BigCodeBench/312": [ + "General", + "Visualization" + ], + "BigCodeBench/313": [ + "System", + "Time", + "General", + "System" + ], + "BigCodeBench/314": [ + "Network", + "Network", + "Network" + ], + "BigCodeBench/315": [ + "Network", + "Network", + "System" + ], + "BigCodeBench/316": [ + "Computation", + "General" + ], + "BigCodeBench/317": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/318": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/319": [ + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/320": [ + "System", + "General", + "System" + ], + "BigCodeBench/321": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/322": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/323": [ + "Computation", + "General", + "General", + "Computation" + ], + "BigCodeBench/324": [ + "System", + "System", + "Time" + ], + "BigCodeBench/325": [ + "System", + "System", + "General", + "System" + ], + "BigCodeBench/326": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/327": [ + "System", + "General", + "General" + ], + "BigCodeBench/328": [ + "General", + "General", + "General" + ], + "BigCodeBench/329": [ + "System", + "General", + "System" + ], + "BigCodeBench/330": [ + "General", + "General" + ], + "BigCodeBench/331": [ + "General", + "General" + ], + "BigCodeBench/332": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/333": [ + "General", + "General" + ], + "BigCodeBench/334": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/335": [ + "General", + "General", + "General" + ], + "BigCodeBench/336": [ + "System", + "System", + "General", + "System" + ], + "BigCodeBench/337": [ + "Computation", + "Visualization" + ], + "BigCodeBench/338": [ + "General", + "Visualization", + "General" + ], + "BigCodeBench/339": [ + "Network", + "Cryptography", + "Cryptography", + "System" + ], + "BigCodeBench/340": [ + "Cryptography", + "Cryptography", + "System" + ], + "BigCodeBench/341": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/342": [ + "General", + "General", + "General" + ], + "BigCodeBench/343": [ + "Computation", + "Visualization" + ], + "BigCodeBench/344": [ + "System", + "System" + ], + "BigCodeBench/345": [ + "Computation", + "Visualization" + ], + "BigCodeBench/346": [ + "System", + "Time", + "System", + "System" + ], + "BigCodeBench/347": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/348": [ + "System", + "Time", + "System", + "System" + ], + "BigCodeBench/349": [ + "Computation", + "General" + ], + "BigCodeBench/350": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/351": [ + "Computation", + "General" + ], + "BigCodeBench/352": [ + "Computation", + "General" + ], + "BigCodeBench/353": [ + "Computation", + "General" + ], + "BigCodeBench/354": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/355": [ + "Computation", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/356": [ + "Visualization", + "Computation", + "Computation" + ], + "BigCodeBench/357": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/358": [ + "General", + "System" + ], + "BigCodeBench/359": [ + "Visualization", + "Computation" + ], + "BigCodeBench/360": [ + "Computation", + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/361": [ + "Computation", + "System" + ], + "BigCodeBench/362": [ + "Computation", + "System" + ], + "BigCodeBench/363": [ + "Computation", + "System" + ], + "BigCodeBench/364": [ + "Computation", + "Computation" + ], + "BigCodeBench/365": [ + "General", + "General", + "System" + ], + "BigCodeBench/366": [ + "General", + "Visualization" + ], + "BigCodeBench/367": [ + "Time", + "General", + "Visualization" + ], + "BigCodeBench/368": [ + "System", + "General", + "System" + ], + "BigCodeBench/369": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/370": [ + "System", + "System", + "General", + "System" + ], + "BigCodeBench/371": [ + "Computation", + "Computation" + ], + "BigCodeBench/372": [ + "System", + "System", + "General" + ], + "BigCodeBench/373": [ + "Visualization", + "Computation" + ], + "BigCodeBench/374": [ + "General", + "System", + "System", + "System" + ], + "BigCodeBench/375": [ + "Visualization", + "Computation" + ], + "BigCodeBench/376": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/377": [ + "System", + "General", + "System" + ], + "BigCodeBench/378": [ + "System", + "Computation", + "General", + "System" + ], + "BigCodeBench/379": [ + "Computation", + "Computation" + ], + "BigCodeBench/380": [ + "System", + "General", + "System" + ], + "BigCodeBench/381": [ + "Computation", + "Visualization", + "Visualization", + "Computation", + "System" + ], + "BigCodeBench/382": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/383": [ + "Computation", + "General", + "Visualization", + "Computation", + "Visualization" + ], + "BigCodeBench/384": [ + "General", + "General", + "General" + ], + "BigCodeBench/385": [ + "General", + "Visualization" + ], + "BigCodeBench/386": [ + "Computation", + "Computation" + ], + "BigCodeBench/387": [ + "Computation", + "Visualization" + ], + "BigCodeBench/388": [ + "Computation", + "General" + ], + "BigCodeBench/389": [ + "System", + "General", + "System" + ], + "BigCodeBench/390": [ + "Computation", + "System", + "Network" + ], + "BigCodeBench/391": [ + "System", + "System", + "System" + ], + "BigCodeBench/392": [ + "Computation", + "Visualization" + ], + "BigCodeBench/393": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/394": [ + "General", + "General", + "General" + ], + "BigCodeBench/395": [ + "System", + "Computation", + "General", + "System", + "General" + ], + "BigCodeBench/396": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/397": [ + "Network", + "General", + "System" + ], + "BigCodeBench/398": [ + "System", + "System" + ], + "BigCodeBench/399": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/400": [ + "System", + "System" + ], + "BigCodeBench/401": [ + "Network", + "Network", + "System" + ], + "BigCodeBench/402": [ + "Network", + "System", + "System", + "General", + "System" + ], + "BigCodeBench/403": [ + "Computation", + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/404": [ + "Computation", + "Computation", + "System" + ], + "BigCodeBench/405": [ + "General", + "Visualization" + ], + "BigCodeBench/406": [ + "Computation", + "Visualization", + "Visualization", + "System" + ], + "BigCodeBench/407": [ + "System", + "System", + "System" + ], + "BigCodeBench/408": [ + "System", + "Computation" + ], + "BigCodeBench/409": [ + "Computation", + "Computation", + "System" + ], + "BigCodeBench/410": [ + "Computation", + "Time", + "System" + ], + "BigCodeBench/411": [ + "Computation", + "System" + ], + "BigCodeBench/412": [ + "Cryptography", + "General", + "System" + ], + "BigCodeBench/413": [ + "Network", + "System" + ], + "BigCodeBench/414": [ + "Computation", + "Computation" + ], + "BigCodeBench/415": [ + "Cryptography", + "Computation" + ], + "BigCodeBench/416": [ + "Computation", + "Visualization" + ], + "BigCodeBench/417": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/418": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/419": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/420": [ + "Computation", + "Computation" + ], + "BigCodeBench/421": [ + "Time", + "System", + "Network", + "System" + ], + "BigCodeBench/422": [ + "Computation", + "Computation" + ], + "BigCodeBench/423": [ + "Computation", + "Computation", + "System" + ], + "BigCodeBench/424": [ + "Computation", + "Computation", + "Computation", + "System" + ], + "BigCodeBench/425": [ + "Visualization", + "Computation", + "System" + ], + "BigCodeBench/426": [ + "Computation", + "Computation", + "System" + ], + "BigCodeBench/427": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/428": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/429": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/430": [ + "Visualization", + "Computation" + ], + "BigCodeBench/431": [ + "Computation", + "Computation", + "System" + ], + "BigCodeBench/432": [ + "Computation", + "Visualization" + ], + "BigCodeBench/433": [ + "Cryptography", + "Cryptography", + "Cryptography", + "Cryptography" + ], + "BigCodeBench/434": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/435": [ + "Computation", + "General" + ], + "BigCodeBench/436": [ + "Visualization", + "General" + ], + "BigCodeBench/437": [ + "System", + "System" + ], + "BigCodeBench/438": [ + "Visualization", + "System", + "System" + ], + "BigCodeBench/439": [ + "Computation", + "Visualization" + ], + "BigCodeBench/440": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/441": [ + "Computation", + "Visualization" + ], + "BigCodeBench/442": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/443": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/444": [ + "Computation", + "Visualization" + ], + "BigCodeBench/445": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/446": [ + "Visualization", + "Computation" + ], + "BigCodeBench/447": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/448": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/449": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/450": [ + "Visualization", + "Computation", + "Computation" + ], + "BigCodeBench/451": [ + "Computation", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/452": [ + "Computation", + "Computation" + ], + "BigCodeBench/453": [ + "General", + "General", + "General" + ], + "BigCodeBench/454": [ + "System", + "System", + "System" + ], + "BigCodeBench/455": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/456": [ + "Computation", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/457": [ + "Computation", + "Computation" + ], + "BigCodeBench/458": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/459": [ + "System", + "Time", + "Time", + "System" + ], + "BigCodeBench/460": [ + "System", + "Computation" + ], + "BigCodeBench/461": [ + "System", + "System", + "System", + "Time" + ], + "BigCodeBench/462": [ + "Computation", + "General" + ], + "BigCodeBench/463": [ + "Computation", + "Computation" + ], + "BigCodeBench/464": [ + "General", + "Time", + "System" + ], + "BigCodeBench/465": [ + "General", + "Time", + "Computation", + "System" + ], + "BigCodeBench/466": [ + "General", + "System" + ], + "BigCodeBench/467": [ + "Computation", + "Visualization" + ], + "BigCodeBench/468": [ + "Computation", + "Computation" + ], + "BigCodeBench/469": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/470": [ + "Computation", + "Visualization" + ], + "BigCodeBench/471": [ + "Computation", + "General" + ], + "BigCodeBench/472": [ + "Visualization", + "Computation" + ], + "BigCodeBench/473": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/474": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/475": [ + "Computation", + "Time" + ], + "BigCodeBench/476": [ + "Visualization", + "Computation" + ], + "BigCodeBench/477": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/478": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/479": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/480": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/481": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/482": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/483": [ + "Computation", + "General" + ], + "BigCodeBench/484": [ + "Computation", + "Computation", + "Time", + "Computation" + ], + "BigCodeBench/485": [ + "Time", + "Time", + "Computation", + "Visualization" + ], + "BigCodeBench/486": [ + "Computation", + "Time", + "Computation" + ], + "BigCodeBench/487": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/488": [ + "Computation", + "Time", + "Computation" + ], + "BigCodeBench/489": [ + "Computation", + "Time", + "General" + ], + "BigCodeBench/490": [ + "Network", + "System" + ], + "BigCodeBench/491": [ + "Time", + "General", + "Visualization" + ], + "BigCodeBench/492": [ + "Computation", + "Time", + "General" + ], + "BigCodeBench/493": [ + "Time", + "General", + "Visualization" + ], + "BigCodeBench/494": [ + "Time", + "General", + "Time", + "General" + ], + "BigCodeBench/495": [ + "Computation", + "Computation" + ], + "BigCodeBench/496": [ + "Time", + "Computation", + "Visualization" + ], + "BigCodeBench/497": [ + "Time", + "Time", + "Time" + ], + "BigCodeBench/498": [ + "Network", + "System" + ], + "BigCodeBench/499": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/500": [ + "System", + "System" + ], + "BigCodeBench/501": [ + "Computation", + "System", + "System" + ], + "BigCodeBench/502": [ + "Computation", + "Time", + "General", + "Visualization" + ], + "BigCodeBench/503": [ + "Computation", + "Time", + "Computation" + ], + "BigCodeBench/504": [ + "Cryptography", + "Cryptography", + "Cryptography" + ], + "BigCodeBench/505": [ + "Cryptography", + "Cryptography" + ], + "BigCodeBench/506": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/507": [ + "Computation", + "Computation" + ], + "BigCodeBench/508": [ + "System", + "Cryptography", + "System" + ], + "BigCodeBench/509": [ + "General", + "Computation", + "System" + ], + "BigCodeBench/510": [ + "General", + "System" + ], + "BigCodeBench/511": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/512": [ + "Computation", + "Computation" + ], + "BigCodeBench/513": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/514": [ + "Computation", + "Visualization" + ], + "BigCodeBench/515": [ + "Computation", + "Visualization" + ], + "BigCodeBench/516": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/517": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/518": [ + "Computation", + "Computation" + ], + "BigCodeBench/519": [ + "Computation", + "Visualization" + ], + "BigCodeBench/520": [ + "General", + "Visualization" + ], + "BigCodeBench/521": [ + "Computation", + "Visualization" + ], + "BigCodeBench/522": [ + "General", + "Visualization" + ], + "BigCodeBench/523": [ + "Computation", + "Visualization" + ], + "BigCodeBench/524": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/525": [ + "General", + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/526": [ + "Computation", + "General", + "Computation", + "System" + ], + "BigCodeBench/527": [ + "Computation", + "General", + "Visualization", + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/528": [ + "Computation", + "System", + "General", + "Visualization" + ], + "BigCodeBench/529": [ + "General", + "Visualization", + "General" + ], + "BigCodeBench/530": [ + "Computation", + "General", + "Visualization", + "Computation", + "Visualization" + ], + "BigCodeBench/531": [ + "General", + "Visualization", + "Computation" + ], + "BigCodeBench/532": [ + "Computation", + "General", + "Visualization", + "Computation" + ], + "BigCodeBench/533": [ + "Cryptography", + "Cryptography", + "Cryptography", + "Computation" + ], + "BigCodeBench/534": [ + "Cryptography", + "Computation", + "Cryptography" + ], + "BigCodeBench/535": [ + "System", + "Computation", + "General" + ], + "BigCodeBench/536": [ + "System", + "Computation", + "System" + ], + "BigCodeBench/537": [ + "System", + "Computation", + "Visualization" + ], + "BigCodeBench/538": [ + "System", + "Computation" + ], + "BigCodeBench/539": [ + "System", + "General", + "System" + ], + "BigCodeBench/540": [ + "Visualization", + "General", + "Computation", + "General" + ], + "BigCodeBench/541": [ + "General", + "General", + "System", + "System" + ], + "BigCodeBench/542": [ + "System", + "Cryptography", + "General" + ], + "BigCodeBench/543": [ + "Cryptography", + "System" + ], + "BigCodeBench/544": [ + "System", + "System" + ], + "BigCodeBench/545": [ + "Cryptography", + "System", + "General" + ], + "BigCodeBench/546": [ + "General", + "General" + ], + "BigCodeBench/547": [ + "Cryptography", + "Cryptography", + "System" + ], + "BigCodeBench/548": [ + "Cryptography", + "General", + "General", + "System" + ], + "BigCodeBench/549": [ + "Computation", + "Cryptography" + ], + "BigCodeBench/550": [ + "Computation", + "General" + ], + "BigCodeBench/551": [ + "Computation", + "General", + "Visualization", + "Visualization" + ], + "BigCodeBench/552": [ + "Visualization", + "General", + "General" + ], + "BigCodeBench/553": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/554": [ + "Computation", + "General" + ], + "BigCodeBench/555": [ + "Computation", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/556": [ + "General", + "Computation", + "General" + ], + "BigCodeBench/557": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/558": [ + "Computation", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/559": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/560": [ + "Computation", + "Time", + "Visualization" + ], + "BigCodeBench/561": [ + "Time", + "Time" + ], + "BigCodeBench/562": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/563": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/564": [ + "Time", + "System", + "System", + "Time" + ], + "BigCodeBench/565": [ + "System", + "Cryptography", + "Cryptography" + ], + "BigCodeBench/566": [ + "General", + "General" + ], + "BigCodeBench/567": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/568": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/569": [ + "General", + "Computation", + "General" + ], + "BigCodeBench/570": [ + "General", + "General", + "System" + ], + "BigCodeBench/571": [ + "Computation", + "General" + ], + "BigCodeBench/572": [ + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/573": [ + "Computation", + "Computation" + ], + "BigCodeBench/574": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/575": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/576": [ + "Computation", + "General" + ], + "BigCodeBench/577": [ + "General", + "Cryptography", + "System", + "System" + ], + "BigCodeBench/578": [ + "General", + "Network" + ], + "BigCodeBench/579": [ + "General", + "System", + "General", + "Visualization" + ], + "BigCodeBench/580": [ + "Computation", + "Computation", + "Visualization", + "Computation", + "General" + ], + "BigCodeBench/581": [ + "Computation", + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/582": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/583": [ + "Cryptography", + "Cryptography", + "Cryptography", + "System" + ], + "BigCodeBench/584": [ + "Network", + "Cryptography", + "Cryptography" + ], + "BigCodeBench/585": [ + "Cryptography", + "Cryptography", + "System", + "System" + ], + "BigCodeBench/586": [ + "Cryptography", + "Cryptography", + "Cryptography" + ], + "BigCodeBench/587": [ + "Cryptography", + "Cryptography", + "Cryptography", + "System" + ], + "BigCodeBench/588": [ + "Computation", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/589": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/590": [ + "Computation", + "Network", + "Time", + "Network" + ], + "BigCodeBench/591": [ + "Computation", + "Time", + "General", + "Visualization" + ], + "BigCodeBench/592": [ + "System", + "Time", + "General", + "System" + ], + "BigCodeBench/593": [ + "Computation", + "Visualization", + "General", + "System", + "System", + "Time" + ], + "BigCodeBench/594": [ + "System", + "General", + "System", + "System", + "Time" + ], + "BigCodeBench/595": [ + "General", + "General", + "General" + ], + "BigCodeBench/596": [ + "Time", + "General", + "Visualization", + "Time" + ], + "BigCodeBench/597": [ + "Computation", + "Time" + ], + "BigCodeBench/598": [ + "Computation", + "Time" + ], + "BigCodeBench/599": [ + "Computation", + "Time" + ], + "BigCodeBench/600": [ + "Computation", + "Computation" + ], + "BigCodeBench/601": [ + "Time", + "Visualization" + ], + "BigCodeBench/602": [ + "Computation", + "Computation" + ], + "BigCodeBench/603": [ + "Computation", + "Computation" + ], + "BigCodeBench/604": [ + "System", + "System" + ], + "BigCodeBench/605": [ + "Computation", + "Visualization" + ], + "BigCodeBench/606": [ + "Computation", + "Computation" + ], + "BigCodeBench/607": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/608": [ + "General", + "Visualization" + ], + "BigCodeBench/609": [ + "General", + "General" + ], + "BigCodeBench/610": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/611": [ + "General", + "Visualization" + ], + "BigCodeBench/612": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/613": [ + "Computation", + "Visualization" + ], + "BigCodeBench/614": [ + "Computation", + "Visualization" + ], + "BigCodeBench/615": [ + "Computation", + "General" + ], + "BigCodeBench/616": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/617": [ + "Computation", + "General", + "Visualization", + "General" + ], + "BigCodeBench/618": [ + "Computation", + "General", + "Visualization", + "Visualization" + ], + "BigCodeBench/619": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/620": [ + "Computation", + "Computation" + ], + "BigCodeBench/621": [ + "Visualization", + "Computation", + "General", + "Computation" + ], + "BigCodeBench/622": [ + "Visualization", + "Computation", + "General", + "Computation" + ], + "BigCodeBench/623": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/624": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/625": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/626": [ + "Time", + "Time", + "General" + ], + "BigCodeBench/627": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/628": [ + "Computation", + "General", + "Visualization" + ], + "BigCodeBench/629": [ + "System", + "Time" + ], + "BigCodeBench/630": [ + "Computation", + "System" + ], + "BigCodeBench/631": [ + "System", + "System" + ], + "BigCodeBench/632": [ + "Computation", + "Time" + ], + "BigCodeBench/633": [ + "Computation", + "General" + ], + "BigCodeBench/634": [ + "General", + "General", + "Computation" + ], + "BigCodeBench/635": [ + "Computation", + "Computation", + "Visualization", + "Computation", + "Computation", + "General" + ], + "BigCodeBench/636": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/637": [ + "Computation", + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/638": [ + "Computation", + "Computation" + ], + "BigCodeBench/639": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/640": [ + "Computation", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/641": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/642": [ + "Cryptography", + "General", + "Cryptography" + ], + "BigCodeBench/643": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/644": [ + "Cryptography", + "Cryptography" + ], + "BigCodeBench/645": [ + "Computation", + "System" + ], + "BigCodeBench/646": [ + "Time", + "Computation", + "System" + ], + "BigCodeBench/647": [ + "Time", + "Time" + ], + "BigCodeBench/648": [ + "Time", + "Time" + ], + "BigCodeBench/649": [ + "Time", + "Computation", + "Computation" + ], + "BigCodeBench/650": [ + "Time", + "Time", + "Time" + ], + "BigCodeBench/651": [ + "Computation", + "Time" + ], + "BigCodeBench/652": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/653": [ + "Visualization", + "Visualization" + ], + "BigCodeBench/654": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/655": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/656": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/657": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/658": [ + "Computation", + "Computation", + "General", + "Computation" + ], + "BigCodeBench/659": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/660": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/661": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/662": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/663": [ + "Computation", + "Computation" + ], + "BigCodeBench/664": [ + "Computation", + "Visualization" + ], + "BigCodeBench/665": [ + "System", + "General", + "General", + "System" + ], + "BigCodeBench/666": [ + "Computation", + "General" + ], + "BigCodeBench/667": [ + "General", + "General" + ], + "BigCodeBench/668": [ + "Computation", + "General" + ], + "BigCodeBench/669": [ + "Computation", + "General" + ], + "BigCodeBench/670": [ + "Computation", + "General" + ], + "BigCodeBench/671": [ + "System", + "General", + "System" + ], + "BigCodeBench/672": [ + "System", + "System" + ], + "BigCodeBench/673": [ + "General", + "System" + ], + "BigCodeBench/674": [ + "Computation", + "System" + ], + "BigCodeBench/675": [ + "General", + "System" + ], + "BigCodeBench/676": [ + "Computation", + "General" + ], + "BigCodeBench/677": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/678": [ + "Computation", + "System", + "System", + "System" + ], + "BigCodeBench/679": [ + "Computation", + "General" + ], + "BigCodeBench/680": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/681": [ + "Computation", + "System" + ], + "BigCodeBench/682": [ + "Computation", + "General" + ], + "BigCodeBench/683": [ + "Computation", + "System" + ], + "BigCodeBench/684": [ + "Computation", + "Computation" + ], + "BigCodeBench/685": [ + "General", + "General" + ], + "BigCodeBench/686": [ + "Computation", + "Computation" + ], + "BigCodeBench/687": [ + "Computation", + "Computation" + ], + "BigCodeBench/688": [ + "Computation", + "Computation" + ], + "BigCodeBench/689": [ + "Computation", + "Computation" + ], + "BigCodeBench/690": [ + "Computation", + "Computation" + ], + "BigCodeBench/691": [ + "Computation", + "Computation" + ], + "BigCodeBench/692": [ + "Computation", + "Computation" + ], + "BigCodeBench/693": [ + "Computation", + "Computation" + ], + "BigCodeBench/694": [ + "General", + "General" + ], + "BigCodeBench/695": [ + "Computation", + "Computation" + ], + "BigCodeBench/696": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/697": [ + "Computation", + "Computation" + ], + "BigCodeBench/698": [ + "Computation", + "Computation" + ], + "BigCodeBench/699": [ + "Computation", + "Computation" + ], + "BigCodeBench/700": [ + "Computation", + "Computation" + ], + "BigCodeBench/701": [ + "Computation", + "Computation" + ], + "BigCodeBench/702": [ + "Computation", + "Computation" + ], + "BigCodeBench/703": [ + "Computation", + "Computation" + ], + "BigCodeBench/704": [ + "Computation", + "General" + ], + "BigCodeBench/705": [ + "Computation", + "Computation" + ], + "BigCodeBench/706": [ + "Computation", + "Computation" + ], + "BigCodeBench/707": [ + "Computation", + "System" + ], + "BigCodeBench/708": [ + "System", + "Cryptography", + "System", + "System" + ], + "BigCodeBench/709": [ + "Network", + "Cryptography", + "General", + "General" + ], + "BigCodeBench/710": [ + "Computation", + "Computation" + ], + "BigCodeBench/711": [ + "System", + "System" + ], + "BigCodeBench/712": [ + "System", + "System", + "System" + ], + "BigCodeBench/713": [ + "General", + "System" + ], + "BigCodeBench/714": [ + "System", + "System" + ], + "BigCodeBench/715": [ + "System", + "System" + ], + "BigCodeBench/716": [ + "Time", + "System", + "System" + ], + "BigCodeBench/717": [ + "System", + "System" + ], + "BigCodeBench/718": [ + "General", + "Computation", + "Computation" + ], + "BigCodeBench/719": [ + "System", + "General", + "System" + ], + "BigCodeBench/720": [ + "System", + "Time", + "General", + "System" + ], + "BigCodeBench/721": [ + "System", + "General", + "System" + ], + "BigCodeBench/722": [ + "Network", + "General", + "System" + ], + "BigCodeBench/723": [ + "System", + "Network", + "Network", + "System" + ], + "BigCodeBench/724": [ + "System", + "System" + ], + "BigCodeBench/725": [ + "Cryptography", + "System", + "System" + ], + "BigCodeBench/726": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/727": [ + "Computation", + "General", + "Computation" + ], + "BigCodeBench/728": [ + "System", + "System" + ], + "BigCodeBench/729": [ + "System", + "General", + "General", + "System" + ], + "BigCodeBench/730": [ + "System", + "System" + ], + "BigCodeBench/731": [ + "System", + "System", + "Computation" + ], + "BigCodeBench/732": [ + "Computation", + "General", + "General", + "General" + ], + "BigCodeBench/733": [ + "General", + "General" + ], + "BigCodeBench/734": [ + "Computation", + "General" + ], + "BigCodeBench/735": [ + "Computation", + "General" + ], + "BigCodeBench/736": [ + "Computation", + "Computation" + ], + "BigCodeBench/737": [ + "Computation", + "Computation" + ], + "BigCodeBench/738": [ + "Computation", + "Computation" + ], + "BigCodeBench/739": [ + "System", + "General" + ], + "BigCodeBench/740": [ + "General", + "General" + ], + "BigCodeBench/741": [ + "General", + "General" + ], + "BigCodeBench/742": [ + "Computation", + "Computation" + ], + "BigCodeBench/743": [ + "System", + "System" + ], + "BigCodeBench/744": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/745": [ + "System", + "General" + ], + "BigCodeBench/746": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/747": [ + "Computation", + "General" + ], + "BigCodeBench/748": [ + "Computation", + "Computation" + ], + "BigCodeBench/749": [ + "Computation", + "Computation" + ], + "BigCodeBench/750": [ + "Computation", + "Computation" + ], + "BigCodeBench/751": [ + "General", + "General" + ], + "BigCodeBench/752": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/753": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/754": [ + "Computation", + "Time", + "Computation" + ], + "BigCodeBench/755": [ + "System", + "System" + ], + "BigCodeBench/756": [ + "General", + "System", + "System" + ], + "BigCodeBench/757": [ + "Time", + "Computation" + ], + "BigCodeBench/758": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/759": [ + "System", + "General", + "System" + ], + "BigCodeBench/760": [ + "Computation", + "Computation", + "Cryptography", + "General", + "Time" + ], + "BigCodeBench/761": [ + "General", + "General", + "System" + ], + "BigCodeBench/762": [ + "Cryptography", + "System", + "System" + ], + "BigCodeBench/763": [ + "General", + "System", + "Computation", + "System" + ], + "BigCodeBench/764": [ + "System", + "General" + ], + "BigCodeBench/765": [ + "System", + "System", + "System" + ], + "BigCodeBench/766": [ + "General", + "General" + ], + "BigCodeBench/767": [ + "General", + "General", + "General" + ], + "BigCodeBench/768": [ + "System", + "General", + "System" + ], + "BigCodeBench/769": [ + "General", + "General", + "General" + ], + "BigCodeBench/770": [ + "Computation", + "Computation" + ], + "BigCodeBench/771": [ + "System", + "System", + "General", + "System" + ], + "BigCodeBench/772": [ + "Computation", + "Computation" + ], + "BigCodeBench/773": [ + "System", + "General", + "System" + ], + "BigCodeBench/774": [ + "Computation", + "Computation" + ], + "BigCodeBench/775": [ + "General", + "General", + "General" + ], + "BigCodeBench/776": [ + "Computation", + "Computation" + ], + "BigCodeBench/777": [ + "System", + "General", + "System" + ], + "BigCodeBench/778": [ + "General", + "General", + "General" + ], + "BigCodeBench/779": [ + "System", + "System" + ], + "BigCodeBench/780": [ + "Time", + "Computation" + ], + "BigCodeBench/781": [ + "Time", + "System" + ], + "BigCodeBench/782": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/783": [ + "System", + "System" + ], + "BigCodeBench/784": [ + "Computation", + "System", + "General" + ], + "BigCodeBench/785": [ + "System", + "System", + "System" + ], + "BigCodeBench/786": [ + "Computation", + "System", + "General" + ], + "BigCodeBench/787": [ + "Computation", + "General" + ], + "BigCodeBench/788": [ + "Computation", + "General" + ], + "BigCodeBench/789": [ + "Computation", + "Computation" + ], + "BigCodeBench/790": [ + "Computation", + "General" + ], + "BigCodeBench/791": [ + "General", + "General", + "General" + ], + "BigCodeBench/792": [ + "Computation", + "General" + ], + "BigCodeBench/793": [ + "Computation", + "General" + ], + "BigCodeBench/794": [ + "General", + "General" + ], + "BigCodeBench/795": [ + "Computation", + "General" + ], + "BigCodeBench/796": [ + "General", + "System" + ], + "BigCodeBench/797": [ + "Computation", + "General" + ], + "BigCodeBench/798": [ + "System", + "System" + ], + "BigCodeBench/799": [ + "Computation", + "General" + ], + "BigCodeBench/800": [ + "System", + "General", + "System" + ], + "BigCodeBench/801": [ + "General", + "Computation" + ], + "BigCodeBench/802": [ + "Computation", + "General" + ], + "BigCodeBench/803": [ + "Computation", + "Computation" + ], + "BigCodeBench/804": [ + "Time", + "System" + ], + "BigCodeBench/805": [ + "Computation", + "General" + ], + "BigCodeBench/806": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/807": [ + "Computation", + "Computation" + ], + "BigCodeBench/808": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/809": [ + "Computation", + "Computation" + ], + "BigCodeBench/810": [ + "System", + "General", + "System" + ], + "BigCodeBench/811": [ + "Computation", + "General" + ], + "BigCodeBench/812": [ + "System", + "System", + "General" + ], + "BigCodeBench/813": [ + "Computation", + "General" + ], + "BigCodeBench/814": [ + "System", + "General", + "System" + ], + "BigCodeBench/815": [ + "Computation", + "Computation" + ], + "BigCodeBench/816": [ + "General", + "General" + ], + "BigCodeBench/817": [ + "General", + "System" + ], + "BigCodeBench/818": [ + "General", + "General" + ], + "BigCodeBench/819": [ + "General", + "Time" + ], + "BigCodeBench/820": [ + "General", + "General" + ], + "BigCodeBench/821": [ + "System", + "Time" + ], + "BigCodeBench/822": [ + "General", + "General" + ], + "BigCodeBench/823": [ + "Computation", + "Time" + ], + "BigCodeBench/824": [ + "General", + "General" + ], + "BigCodeBench/825": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/826": [ + "System", + "General", + "System" + ], + "BigCodeBench/827": [ + "Computation", + "Computation" + ], + "BigCodeBench/828": [ + "System", + "System", + "System" + ], + "BigCodeBench/829": [ + "Computation", + "Computation" + ], + "BigCodeBench/830": [ + "System", + "System" + ], + "BigCodeBench/831": [ + "Computation", + "General" + ], + "BigCodeBench/832": [ + "System", + "System" + ], + "BigCodeBench/833": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/834": [ + "System", + "System", + "Cryptography" + ], + "BigCodeBench/835": [ + "Computation", + "Computation" + ], + "BigCodeBench/836": [ + "System", + "System", + "System" + ], + "BigCodeBench/837": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/838": [ + "Computation", + "General" + ], + "BigCodeBench/839": [ + "System", + "General" + ], + "BigCodeBench/840": [ + "Computation", + "Computation" + ], + "BigCodeBench/841": [ + "General", + "General", + "General", + "System" + ], + "BigCodeBench/842": [ + "System", + "General" + ], + "BigCodeBench/843": [ + "General", + "General" + ], + "BigCodeBench/844": [ + "General", + "System", + "General" + ], + "BigCodeBench/845": [ + "General", + "General", + "Computation", + "Computation" + ], + "BigCodeBench/846": [ + "Computation", + "General" + ], + "BigCodeBench/847": [ + "General", + "General", + "General", + "System" + ], + "BigCodeBench/848": [ + "General", + "General" + ], + "BigCodeBench/849": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/850": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/851": [ + "General", + "General" + ], + "BigCodeBench/852": [ + "General", + "General" + ], + "BigCodeBench/853": [ + "System", + "General", + "System" + ], + "BigCodeBench/854": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/855": [ + "General", + "General", + "General" + ], + "BigCodeBench/856": [ + "General", + "Computation", + "General" + ], + "BigCodeBench/857": [ + "System", + "General", + "Time", + "System", + "System" + ], + "BigCodeBench/858": [ + "General", + "General", + "General" + ], + "BigCodeBench/859": [ + "General", + "Computation" + ], + "BigCodeBench/860": [ + "General", + "General", + "General" + ], + "BigCodeBench/861": [ + "General", + "General" + ], + "BigCodeBench/862": [ + "General", + "General", + "General" + ], + "BigCodeBench/863": [ + "Computation", + "Computation" + ], + "BigCodeBench/864": [ + "Computation", + "Computation" + ], + "BigCodeBench/865": [ + "Computation", + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/866": [ + "Computation", + "Computation" + ], + "BigCodeBench/867": [ + "General", + "General" + ], + "BigCodeBench/868": [ + "General", + "General" + ], + "BigCodeBench/869": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/870": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/871": [ + "Computation", + "General" + ], + "BigCodeBench/872": [ + "Computation", + "General" + ], + "BigCodeBench/873": [ + "System", + "System" + ], + "BigCodeBench/874": [ + "General", + "Computation" + ], + "BigCodeBench/875": [ + "Computation", + "General" + ], + "BigCodeBench/876": [ + "General", + "System", + "General", + "System" + ], + "BigCodeBench/877": [ + "Computation", + "Computation" + ], + "BigCodeBench/878": [ + "Computation", + "Computation" + ], + "BigCodeBench/879": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/880": [ + "Computation", + "Computation" + ], + "BigCodeBench/881": [ + "Computation", + "General" + ], + "BigCodeBench/882": [ + "System", + "Computation", + "System" + ], + "BigCodeBench/883": [ + "Computation", + "Computation" + ], + "BigCodeBench/884": [ + "Computation", + "Computation" + ], + "BigCodeBench/885": [ + "Computation", + "Computation" + ], + "BigCodeBench/886": [ + "Computation", + "General" + ], + "BigCodeBench/887": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/888": [ + "Computation", + "System" + ], + "BigCodeBench/889": [ + "Computation", + "Computation", + "System" + ], + "BigCodeBench/890": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/891": [ + "Computation", + "Computation" + ], + "BigCodeBench/892": [ + "General", + "General" + ], + "BigCodeBench/893": [ + "Time", + "General" + ], + "BigCodeBench/894": [ + "Computation", + "Visualization" + ], + "BigCodeBench/895": [ + "Computation", + "Visualization" + ], + "BigCodeBench/896": [ + "General", + "General", + "General" + ], + "BigCodeBench/897": [ + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/898": [ + "General", + "General" + ], + "BigCodeBench/899": [ + "Computation", + "General" + ], + "BigCodeBench/900": [ + "Computation", + "Computation" + ], + "BigCodeBench/901": [ + "Computation", + "Computation" + ], + "BigCodeBench/902": [ + "Computation", + "General" + ], + "BigCodeBench/903": [ + "Computation", + "Computation" + ], + "BigCodeBench/904": [ + "Computation", + "Visualization" + ], + "BigCodeBench/905": [ + "System", + "System", + "System" + ], + "BigCodeBench/906": [ + "General", + "System", + "System", + "System" + ], + "BigCodeBench/907": [ + "General", + "System" + ], + "BigCodeBench/908": [ + "Computation", + "Visualization", + "General", + "System" + ], + "BigCodeBench/909": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/910": [ + "Computation", + "Visualization" + ], + "BigCodeBench/911": [ + "General", + "General", + "General" + ], + "BigCodeBench/912": [ + "General", + "General" + ], + "BigCodeBench/913": [ + "General", + "Computation", + "Computation" + ], + "BigCodeBench/914": [ + "Computation", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/915": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/916": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/917": [ + "Computation", + "General", + "Visualization", + "Computation" + ], + "BigCodeBench/918": [ + "Computation", + "General" + ], + "BigCodeBench/919": [ + "Computation", + "Visualization" + ], + "BigCodeBench/920": [ + "Computation", + "Visualization" + ], + "BigCodeBench/921": [ + "Computation", + "Computation" + ], + "BigCodeBench/922": [ + "Computation", + "General" + ], + "BigCodeBench/923": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/924": [ + "Computation", + "System", + "System" + ], + "BigCodeBench/925": [ + "Computation", + "Computation" + ], + "BigCodeBench/926": [ + "System", + "Computation" + ], + "BigCodeBench/927": [ + "Computation", + "Computation" + ], + "BigCodeBench/928": [ + "General", + "General", + "General" + ], + "BigCodeBench/929": [ + "Computation", + "Computation" + ], + "BigCodeBench/930": [ + "General", + "General" + ], + "BigCodeBench/931": [ + "General", + "General" + ], + "BigCodeBench/932": [ + "General", + "General" + ], + "BigCodeBench/933": [ + "Computation", + "General" + ], + "BigCodeBench/934": [ + "Cryptography", + "General" + ], + "BigCodeBench/935": [ + "Computation", + "General" + ], + "BigCodeBench/936": [ + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/937": [ + "General", + "General" + ], + "BigCodeBench/938": [ + "Computation", + "General" + ], + "BigCodeBench/939": [ + "System", + "General", + "System" + ], + "BigCodeBench/940": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/941": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/942": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/943": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/944": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/945": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/946": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/947": [ + "Time", + "Computation", + "General" + ], + "BigCodeBench/948": [ + "Computation", + "Computation" + ], + "BigCodeBench/949": [ + "Computation", + "Computation" + ], + "BigCodeBench/950": [ + "Computation", + "Computation" + ], + "BigCodeBench/951": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/952": [ + "Computation", + "Time", + "General" + ], + "BigCodeBench/953": [ + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/954": [ + "General", + "General" + ], + "BigCodeBench/955": [ + "General", + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/956": [ + "General", + "General", + "General" + ], + "BigCodeBench/957": [ + "General", + "General" + ], + "BigCodeBench/958": [ + "General", + "General" + ], + "BigCodeBench/959": [ + "General", + "General" + ], + "BigCodeBench/960": [ + "General", + "General" + ], + "BigCodeBench/961": [ + "System", + "General", + "System" + ], + "BigCodeBench/962": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/963": [ + "System", + "System", + "System", + "System" + ], + "BigCodeBench/964": [ + "Computation", + "System", + "System", + "System" + ], + "BigCodeBench/965": [ + "System", + "General", + "System" + ], + "BigCodeBench/966": [ + "Computation", + "Visualization" + ], + "BigCodeBench/967": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/968": [ + "Computation", + "Visualization" + ], + "BigCodeBench/969": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/970": [ + "Computation", + "Visualization" + ], + "BigCodeBench/971": [ + "Time", + "System", + "System" + ], + "BigCodeBench/972": [ + "System", + "System" + ], + "BigCodeBench/973": [ + "System", + "System" + ], + "BigCodeBench/974": [ + "System", + "System" + ], + "BigCodeBench/975": [ + "Computation", + "Computation" + ], + "BigCodeBench/976": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/977": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/978": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/979": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/980": [ + "Computation", + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/981": [ + "Computation", + "Time", + "General" + ], + "BigCodeBench/982": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/983": [ + "Computation", + "Visualization" + ], + "BigCodeBench/984": [ + "Visualization", + "Computation" + ], + "BigCodeBench/985": [ + "Computation", + "Computation", + "System", + "System" + ], + "BigCodeBench/986": [ + "Computation", + "Visualization", + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/987": [ + "Computation", + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/988": [ + "System", + "General", + "System" + ], + "BigCodeBench/989": [ + "General", + "General" + ], + "BigCodeBench/990": [ + "Cryptography", + "Network", + "Cryptography", + "Cryptography" + ], + "BigCodeBench/991": [ + "General", + "General", + "Cryptography" + ], + "BigCodeBench/992": [ + "System", + "System" + ], + "BigCodeBench/993": [ + "General", + "Visualization", + "Computation" + ], + "BigCodeBench/994": [ + "Computation", + "Network", + "Network" + ], + "BigCodeBench/995": [ + "Computation", + "Computation", + "Visualization", + "System" + ], + "BigCodeBench/996": [ + "Network", + "Network", + "System" + ], + "BigCodeBench/997": [ + "Network", + "System", + "System" + ], + "BigCodeBench/998": [ + "System", + "Network", + "Cryptography", + "System" + ], + "BigCodeBench/999": [ + "Network", + "System", + "General", + "System" + ], + "BigCodeBench/1000": [ + "Computation", + "Network", + "System", + "System" + ], + "BigCodeBench/1001": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1002": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1003": [ + "Computation", + "Network", + "Network" + ], + "BigCodeBench/1004": [ + "Network", + "General", + "Visualization", + "General" + ], + "BigCodeBench/1005": [ + "Network", + "System", + "System" + ], + "BigCodeBench/1006": [ + "System", + "Network", + "System" + ], + "BigCodeBench/1007": [ + "Computation", + "Network" + ], + "BigCodeBench/1008": [ + "Computation", + "System", + "Network", + "Network" + ], + "BigCodeBench/1009": [ + "Network", + "System" + ], + "BigCodeBench/1010": [ + "System", + "Visualization", + "Network" + ], + "BigCodeBench/1011": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1012": [ + "System", + "Network", + "System" + ], + "BigCodeBench/1013": [ + "System", + "Network", + "Network", + "Network" + ], + "BigCodeBench/1014": [ + "Computation", + "Network" + ], + "BigCodeBench/1015": [ + "System", + "Computation", + "Network", + "Network" + ], + "BigCodeBench/1016": [ + "Visualization", + "Computation", + "Visualization", + "Network" + ], + "BigCodeBench/1017": [ + "Computation", + "Computation" + ], + "BigCodeBench/1018": [ + "Network", + "Network" + ], + "BigCodeBench/1019": [ + "Cryptography", + "Computation", + "Visualization" + ], + "BigCodeBench/1020": [ + "Network", + "Network", + "System" + ], + "BigCodeBench/1021": [ + "Cryptography", + "Cryptography" + ], + "BigCodeBench/1022": [ + "Computation", + "Time", + "System" + ], + "BigCodeBench/1023": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/1024": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/1025": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1026": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1027": [ + "Network", + "Cryptography" + ], + "BigCodeBench/1028": [ + "System", + "System", + "System", + "Time" + ], + "BigCodeBench/1029": [ + "Computation", + "Computation" + ], + "BigCodeBench/1030": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/1031": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/1032": [ + "Computation", + "Visualization", + "General", + "General", + "Visualization" + ], + "BigCodeBench/1033": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/1034": [ + "Computation", + "Computation" + ], + "BigCodeBench/1035": [ + "Computation", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1036": [ + "Computation", + "Visualization", + "Visualization" + ], + "BigCodeBench/1037": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1038": [ + "Time", + "System" + ], + "BigCodeBench/1039": [ + "Cryptography", + "System", + "Network" + ], + "BigCodeBench/1040": [ + "General", + "Time", + "System", + "Network" + ], + "BigCodeBench/1041": [ + "General", + "System" + ], + "BigCodeBench/1042": [ + "Network", + "Network", + "System" + ], + "BigCodeBench/1043": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1044": [ + "Computation", + "Time" + ], + "BigCodeBench/1045": [ + "Time", + "Time", + "Computation" + ], + "BigCodeBench/1046": [ + "Computation", + "Time", + "General" + ], + "BigCodeBench/1047": [ + "Time", + "General", + "Visualization" + ], + "BigCodeBench/1048": [ + "Time", + "Computation", + "Visualization" + ], + "BigCodeBench/1049": [ + "Computation", + "General" + ], + "BigCodeBench/1050": [ + "Cryptography", + "System" + ], + "BigCodeBench/1051": [ + "General", + "Computation", + "Visualization" + ], + "BigCodeBench/1052": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1053": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1054": [ + "System", + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1055": [ + "Computation", + "General", + "General" + ], + "BigCodeBench/1056": [ + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/1057": [ + "Computation", + "Computation", + "General" + ], + "BigCodeBench/1058": [ + "Visualization", + "General", + "Visualization" + ], + "BigCodeBench/1059": [ + "Computation", + "Computation", + "General", + "General" + ], + "BigCodeBench/1060": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1061": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1062": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1063": [ + "Visualization", + "Computation" + ], + "BigCodeBench/1064": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1065": [ + "Visualization", + "Computation" + ], + "BigCodeBench/1066": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1067": [ + "System", + "Network" + ], + "BigCodeBench/1068": [ + "System", + "Computation", + "General" + ], + "BigCodeBench/1069": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1070": [ + "Computation", + "General" + ], + "BigCodeBench/1071": [ + "General", + "Computation", + "Visualization", + "General" + ], + "BigCodeBench/1072": [ + "Computation", + "Computation" + ], + "BigCodeBench/1073": [ + "Visualization", + "Time" + ], + "BigCodeBench/1074": [ + "Time", + "Time" + ], + "BigCodeBench/1075": [ + "Time", + "Computation", + "Visualization" + ], + "BigCodeBench/1076": [ + "Time", + "Computation", + "Time" + ], + "BigCodeBench/1077": [ + "Time", + "Time", + "Computation" + ], + "BigCodeBench/1078": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1079": [ + "Computation", + "Computation", + "Visualization" + ], + "BigCodeBench/1080": [ + "Computation", + "Computation" + ], + "BigCodeBench/1081": [ + "Computation", + "Visualization" + ], + "BigCodeBench/1082": [ + "Computation", + "Computation" + ], + "BigCodeBench/1083": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1084": [ + "Computation", + "Computation" + ], + "BigCodeBench/1085": [ + "General", + "Visualization", + "General" + ], + "BigCodeBench/1086": [ + "Computation", + "Computation", + "General", + "General" + ], + "BigCodeBench/1087": [ + "Computation", + "Visualization", + "Computation" + ], + "BigCodeBench/1088": [ + "Computation", + "Computation", + "Computation" + ], + "BigCodeBench/1089": [ + "General", + "Computation" + ], + "BigCodeBench/1090": [ + "General", + "General", + "System" + ], + "BigCodeBench/1091": [ + "System", + "General", + "System" + ], + "BigCodeBench/1092": [ + "General", + "Network", + "Network" + ], + "BigCodeBench/1093": [ + "General", + "General" + ], + "BigCodeBench/1094": [ + "Computation", + "General" + ], + "BigCodeBench/1095": [ + "Computation", + "General", + "System" + ], + "BigCodeBench/1096": [ + "Computation", + "System", + "General", + "System" + ], + "BigCodeBench/1097": [ + "General", + "General" + ], + "BigCodeBench/1098": [ + "General", + "General" + ], + "BigCodeBench/1099": [ + "General", + "General" + ], + "BigCodeBench/1100": [ + "General", + "Computation" + ], + "BigCodeBench/1101": [ + "System", + "System", + "System", + "Time" + ], + "BigCodeBench/1102": [ + "System", + "Time", + "System" + ], + "BigCodeBench/1103": [ + "System", + "System", + "System" + ], + "BigCodeBench/1104": [ + "System", + "System", + "System" + ], + "BigCodeBench/1105": [ + "System", + "System", + "System", + "Time" + ], + "BigCodeBench/1106": [ + "Time", + "System", + "System" + ], + "BigCodeBench/1107": [ + "Time", + "Time" + ], + "BigCodeBench/1108": [ + "General", + "General" + ], + "BigCodeBench/1109": [ + "Computation", + "System" + ], + "BigCodeBench/1110": [ + "General", + "General", + "General" + ], + "BigCodeBench/1111": [ + "General", + "General", + "General" + ], + "BigCodeBench/1112": [ + "System", + "General" + ], + "BigCodeBench/1113": [ + "System", + "General" + ], + "BigCodeBench/1114": [ + "General", + "General" + ], + "BigCodeBench/1115": [ + "General", + "General" + ], + "BigCodeBench/1116": [ + "Computation", + "General" + ], + "BigCodeBench/1117": [ + "General", + "General", + "System" + ], + "BigCodeBench/1118": [ + "System", + "System", + "Network", + "System" + ], + "BigCodeBench/1119": [ + "Cryptography", + "Cryptography", + "General", + "General" + ], + "BigCodeBench/1120": [ + "Network", + "General", + "Network", + "System" + ], + "BigCodeBench/1121": [ + "Network", + "General", + "Network", + "System" + ], + "BigCodeBench/1122": [ + "Network", + "Network", + "General" + ], + "BigCodeBench/1123": [ + "Network", + "General", + "Network", + "Network" + ], + "BigCodeBench/1124": [ + "General", + "Network", + "Network", + "Network" + ], + "BigCodeBench/1125": [ + "Network", + "General", + "System" + ], + "BigCodeBench/1126": [ + "Cryptography", + "General" + ], + "BigCodeBench/1127": [ + "Cryptography", + "General", + "System" + ], + "BigCodeBench/1128": [ + "Cryptography", + "Cryptography", + "Time", + "System", + "System" + ], + "BigCodeBench/1129": [ + "System", + "Time", + "Network", + "System" + ], + "BigCodeBench/1130": [ + "Cryptography", + "System", + "System", + "System" + ], + "BigCodeBench/1131": [ + "Cryptography", + "Cryptography" + ], + "BigCodeBench/1132": [ + "Cryptography", + "Cryptography", + "System" + ], + "BigCodeBench/1133": [ + "Network", + "System" + ], + "BigCodeBench/1134": [ + "System", + "Cryptography", + "System" + ], + "BigCodeBench/1135": [ + "General", + "Network", + "System" + ], + "BigCodeBench/1136": [ + "System", + "General", + "Network", + "Network" + ], + "BigCodeBench/1137": [ + "General", + "Network", + "Network", + "System" + ], + "BigCodeBench/1138": [ + "Computation", + "General" + ], + "BigCodeBench/1139": [ + "Computation", + "Computation", + "Computation" + ] +} \ No newline at end of file From 80913b074097f9e75ddc4b3be26025d3ba1087a9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 26 Jul 2024 07:21:30 +0800 Subject: [PATCH 094/325] add more models --- analysis/utils.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index 4b1307c..88fc003 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -872,4 +872,103 @@ "act_param": 9, "open-data": "None", }, + "gpt-4o-mini-2024-07-18": { + "name": "GPT-4o-mini-2024-07-18", + "link": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "Nexusflow/Athene-70B": { + "name": "Athene-70B", + "link": "https://huggingface.co/Nexusflow/Athene-70B", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "NTQAI/Nxcode-CQ-7B-orpo": { + "name": "Nxcode-CQ-7B-Orpo", + "link": "https://huggingface.co/NTQAI/Nxcode-CQ-7B-orpo", + "prompted": True, + "moe": False, + "size": 7, + "act_param": 7, + "open-data": "None", + }, + "migtissera/Llama-3-70B-Synthia-v3.5": { + "name": "Llama-3-70B-Synthia-v3.5", + "link": "https://huggingface.co/migtissera/Llama-3-70B-Synthia-v3.5", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "migtissera/Tess-v2.5.2-Qwen2-72B": { + "name": "Tess-v2.5.2-Qwen2-72B", + "link": "https://huggingface.co/migtissera/Tess-v2.5.2-Qwen2-72B", + "prompted": True, + "moe": False, + "size": 72, + "act_param": 72, + "open-data": "None", + }, + "WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5": { + "name": "WhiteRabbitNeo-33B-v1.5", + "link": "https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5", + "prompted": True, + "moe": False, + "size": 33, + "act_param": 33, + "open-data": "None", + }, + "mistral-large-2407": { + "name": "Mistral-Large-Instruct-2407", + "link": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407", + "prompted": True, + "moe": False, + "size": 123, + "act_param": 123, + "open-data": "None", + }, + "meta-llama/Meta-Llama-3.1-8B-Instruct": { + "name": "Llama-3.1-8B-Instruct", + "link": "https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct", + "prompted": True, + "moe": False, + "size": 8, + "act_param": 8, + "open-data": "None", + }, + "meta-llama/Meta-Llama-3.1-70B-Instruct": { + "name": "Llama-3.1-70B-Instruct", + "link": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "meta--llama-3.1-405b-instruct": { + "name": "Llama-3.1-405B-Instruct", + "link": "https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct", + "prompted": True, + "moe": False, + "size": 405, + "act_param": 405, + "open-data": "None", + }, + "deepseek-coder-20240724": { + "name": "DeepSeek-Coder-V2-Instruct (2024-07-24)", + "link": "https://www.deepseek.com/", + "prompted": True, + "moe": True, + "size": 236, + "act_param": 21, + "open-data": "None", + }, } \ No newline at end of file From e61bda03a6ee16f242799a893f888e39e06c1b99 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 26 Jul 2024 07:22:07 +0800 Subject: [PATCH 095/325] fix: compute hard subset only --- analysis/get_results.py | 111 ++++++++++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 33 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index 83af2b3..0df3da2 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -17,11 +17,15 @@ def update_model_info(model_info): if "https://huggingface.co/" in info["link"]: hf_model = info["link"].split("https://huggingface.co/")[-1] print(hf_model) - tokenizer = AutoTokenizer.from_pretrained(hf_model, trust_remote_code=True) - if tokenizer.chat_template is None: + try: + tokenizer = AutoTokenizer.from_pretrained(hf_model, trust_remote_code=True) + + if tokenizer.chat_template is None: + model_info[model]["direct_complete"] = True + else: + model_info[model]["direct_complete"] = False + except: model_info[model]["direct_complete"] = True - else: - model_info[model]["direct_complete"] = False else: model_info[model]["direct_complete"] = False @@ -44,7 +48,7 @@ def get_results(tids): "moe": info["moe"], "size": info["size"], "act_param": info["act_param"], - "direct_complete": info["direct_complete"], + # "direct_complete": info["direct_complete"], } for model, info in model_info.items(): @@ -53,10 +57,16 @@ def get_results(tids): files = glob(f"results/{model}--bigcodebench-*.json") assert files, f"No files found for results/{model}--bigcodebench-*.json" for file in files: - _, suffix = os.path.basename(file).split("--bigcodebench-") + try: + _, suffix = os.path.basename(file).split("--bigcodebench-hard-") + with open("results/"+model+"--bigcodebench-hard-"+suffix, "r") as f: + data = json.load(f) + except: + _, suffix = os.path.basename(file).split("--bigcodebench-") + with open("results/"+model+"--bigcodebench-"+suffix, "r") as f: + data = json.load(f) status = [] - with open("results/"+model+"--bigcodebench-"+suffix, "r") as f: - data = json.load(f) + for key, value in data["eval"].items(): if key not in tids: continue @@ -76,22 +86,22 @@ def get_results(tids): mode = "-cal" results[info["name"]][f"pass@1"][f"{task}{mode}"] = round(mean(status)*100,1) - if not info["prompted"] or info["direct_complete"]: + if not info["prompted"]:# or info["direct_complete"]: results[info["name"]][f"pass@1"][f"{task}-cal"] = round(mean(status)*100,1) for model, result in results.items(): for task in ["complete"]: origin = result["pass@1"].pop(task) - assert origin, f"Missing original complete results for {model}" + # assert origin, f"Missing original complete results for {model}" calibrate = result["pass@1"].pop(f"{task}-cal") if calibrate: - if calibrate - origin > 1: - results[model]["lazy"] = True - else: - results[model]["lazy"] = False + # if calibrate - origin > 1: + # results[model]["lazy"] = True + # else: + # results[model]["lazy"] = False results[model]["pass@1"][task] = calibrate else: - results[model]["lazy"] = False + # results[model]["lazy"] = False results[model]["pass@1"][task] = origin calibrate_instruct = result["pass@1"].pop(f"instruct-cal") result["pass@1"]["instruct"] = calibrate_instruct @@ -151,14 +161,44 @@ def read_task_perf(tids, task="complete"): task_perf = dict() model = model.replace("/", "--") try: - if info["prompted"] and not info["direct_complete"]: - files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") - if files: - file = files[0] - else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] - else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + try: + try: + if info["prompted"]:# and not info["direct_complete"]: + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_hard_eval_results.json") + if files: + file = files[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_hard_eval_results.json")[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_hard_eval_results.json")[0] + except: + if info["prompted"]: + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") + if files: + file = files[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + else: + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + except: + try: + if info["prompted"]:# and not info["direct_complete"]: + files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized-calibrated_hard_eval_results.json") + if files: + file = files[0] + else: + file = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized_hard_eval_results.json")[0] + else: + file = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized_hard_eval_results.json")[0] + except: + if info["prompted"]: + files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized-calibrated_eval_results.json") + if files: + file = files[0] + else: + file = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized_eval_results.json")[0] + else: + file = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized_eval_results.json")[0] except: continue @@ -255,8 +295,9 @@ def get_elo_mle(df, SCALE=400, BASE=10, INIT_RATING=1000): def update_elo_rating(results, elo_dict): for model, info in model_info.items(): if info["name"] not in elo_dict: - continue - results[info["name"]]["elo_mle"] = elo_dict[info["name"]] + results[info["name"]]["elo_mle"] = None + else: + results[info["name"]]["elo_mle"] = elo_dict[info["name"]] return results @@ -296,7 +337,7 @@ def get_solve_rate(data_dict, task="complete"): def get_hf_ds(results): - hf_dataset = {"model": [], "link": [], "moe": [], "size": [], "act_param": [], "type": [], "lazy": [], "direct_complete": [], + hf_dataset = {"model": [], "link": [], "moe": [], "size": [], "act_param": [], "type": [], #"lazy": [],# "direct_complete": [], "complete": [], "instruct": [], "elo_mle": []} for model, result in results.items(): @@ -306,10 +347,10 @@ def get_hf_ds(results): hf_dataset["size"].append(result["size"]) hf_dataset["act_param"].append(result["act_param"]) hf_dataset["type"].append("🔶" if result["prompted"] else "🟢") - hf_dataset["lazy"].append(result["lazy"]) + # hf_dataset["lazy"].append(result["lazy"]) hf_dataset["complete"].append(result["pass@1"]["complete"]) hf_dataset["instruct"].append(result["pass@1"]["instruct"]) - hf_dataset["direct_complete"].append(result["direct_complete"]) + # hf_dataset["direct_complete"].append(result["direct_complete"]) hf_dataset["elo_mle"].append(result["elo_mle"]) return Dataset.from_dict(hf_dataset) @@ -335,11 +376,11 @@ def push_ds(ds, path, local=False): if __name__ == "__main__": - bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.0_hf") + # bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.0_hf") bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.0_hf") - model_info = update_model_info(model_info) + # model_info = update_model_info(model_info) bcb_config = { - "": bcb_orig, + # "": bcb_orig, "-hard": bcb_hard, } for suffix, bcb in bcb_config.items(): @@ -347,7 +388,8 @@ def push_ds(ds, path, local=False): files = [] complete_data, complete_files = read_task_perf(bcb["task_id"], "complete") instruct_data, instruct_files = read_task_perf(bcb["task_id"], "instruct") - assert len(model_info) == len(complete_data) + assert len(model_info) == len(complete_data),\ + f"Missing results for {set([val['name'] for val in model_info.values()]) - set([model for model in complete_data.keys()])}" with open("task2domain.json", "r") as f: task2domain = json.load(f) domain_complete = get_domain_perf(complete_data, task2domain) @@ -372,7 +414,10 @@ def push_ds(ds, path, local=False): } elo_ds = dict() for config, (task_level, no_tie) in elo_config.items(): - battles = get_winner_df(complete_data, bcb["task_id"], "complete", task_level=task_level, no_tie=no_tie) + filter_complete_data = {model: task_perf for model, task_perf in complete_data.items() if model in instruct_data} + complete_battles = get_winner_df(filter_complete_data, bcb["task_id"], "complete", task_level=task_level, no_tie=no_tie) + instruct_battles = get_winner_df(instruct_data, bcb["task_id"], "instruct", task_level=task_level, no_tie=no_tie) + battles = pd.concat([complete_battles, instruct_battles]) elo_mle_bootstrap = get_bootstrap_result(battles, get_elo_mle, 500) bootstrap_lu_median = elo_mle_bootstrap.median().reset_index().set_axis(["model", "Elo rating"], axis=1) bootstrap_lu_median["Elo rating"] = (bootstrap_lu_median["Elo rating"] + 0.5).astype(int) From 19f63daf08234416321a43c3c64fe63a08bd1fe5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 27 Jul 2024 03:23:53 +0800 Subject: [PATCH 096/325] add task perf push --- analysis/get_results.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/analysis/get_results.py b/analysis/get_results.py index 0df3da2..664e156 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -374,6 +374,17 @@ def push_ds(ds, path, local=False): ds.push_to_hub(path) +def get_perf_df(data_dict): + perfs = {"Model": []} + for task_id in data_dict[list(data_dict.keys())[0]]: + perfs[task_id] = [] + for model, task_perf in data_dict.items(): + perfs["Model"].append(model) + for task_id, status in task_perf.items(): + perfs[task_id].append(status) + return pd.DataFrame(perfs) + + if __name__ == "__main__": # bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.0_hf") @@ -388,6 +399,9 @@ def push_ds(ds, path, local=False): files = [] complete_data, complete_files = read_task_perf(bcb["task_id"], "complete") instruct_data, instruct_files = read_task_perf(bcb["task_id"], "instruct") + complete_df = get_perf_df(complete_data) + instruct_df = get_perf_df(instruct_data) + push_ds(DatasetDict({"complete": Dataset.from_pandas(complete_df), "instruct": Dataset.from_pandas(instruct_df)}), f"bigcode/bigcodebench{suffix}-perf") assert len(model_info) == len(complete_data),\ f"Missing results for {set([val['name'] for val in model_info.values()]) - set([model for model in complete_data.keys()])}" with open("task2domain.json", "r") as f: From 4a17a575b6cc55374037c6b00f294607fbc67401 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 29 Jul 2024 22:15:11 +0800 Subject: [PATCH 097/325] fix: adjust the max limit --- bigcodebench/evaluate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 76d8e23..9a3c25e 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -323,14 +323,14 @@ def main(): parser.add_argument( "--split", required=True, type=str, choices=["complete", "instruct"] ) - parser.add_argument("--subset", default="full", type=str, choices=["full", "hard"]) + parser.add_argument("--subset", default="hard", type=str, choices=["full", "hard"]) parser.add_argument("--samples", required=True, type=str) parser.add_argument("--save_pass_rate", action="store_true") parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) - parser.add_argument("--max-as-limit", default=128*1024, type=int) - parser.add_argument("--max-data-limit", default=4*1024, type=int) - parser.add_argument("--max-stack-limit", default=5, type=int) + parser.add_argument("--max-as-limit", default=30*1024, type=int) + parser.add_argument("--max-data-limit", default=30*1024, type=int) + parser.add_argument("--max-stack-limit", default=10, type=int) parser.add_argument( "--check-gt-only", action="store_true", help="Check the ground truth" ) From 81ec0107be12aba65b811daee56b5eee213069ee Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 29 Jul 2024 22:15:33 +0800 Subject: [PATCH 098/325] fix: add min time limit --- bigcodebench/evaluate.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 9a3c25e..4500d3f 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -34,7 +34,7 @@ Result = Tuple[str, List[bool]] -def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit, max_data_limit, max_stack_limit): +def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit, max_data_limit, max_stack_limit, min_time_limit): cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl") if os.path.exists(cache_file): if check_gt_only: @@ -60,7 +60,8 @@ def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit, problem["task_id"], max_as_limit, max_data_limit, - max_stack_limit + max_stack_limit, + min_time_limit, ) futures.append(executor.submit(trusted_check, *args)) @@ -129,7 +130,7 @@ def evaluate(flags): dataset_hash = get_bigcodebench_hash(subset=flags.subset) if not flags.no_gt: - expected_time = get_groundtruth(n_workers, problems, dataset_hash, flags.check_gt_only, flags.max_as_limit, flags.max_data_limit, flags.max_stack_limit) + expected_time = get_groundtruth(n_workers, problems, dataset_hash, flags.check_gt_only, flags.max_as_limit, flags.max_data_limit, flags.max_stack_limit, flags.min_time_limit) else: expected_time = {task_id: None for task_id in problems} From 229bd322aaadbd8613a84db6e7ff64de2ada9310 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 29 Jul 2024 22:15:55 +0800 Subject: [PATCH 099/325] fix: add min time limit --- bigcodebench/gen/util/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bigcodebench/gen/util/__init__.py b/bigcodebench/gen/util/__init__.py index f8f6238..6e841bb 100644 --- a/bigcodebench/gen/util/__init__.py +++ b/bigcodebench/gen/util/__init__.py @@ -54,8 +54,10 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta start = time.time() with safe_environment(), swallow_io(), time_limit(seconds=TIMEOUT_LIMIT): suite.run(test_result) - - if len(test_result.failures + test_result.errors) > 0: + + errors = test_result.failures + test_result.errors + if len(errors) > 0: + print(errors) times.value = -1 else: times.value = time.time() - start @@ -83,8 +85,9 @@ def trusted_check( max_as_limit: float, max_data_limit: float, max_stack_limit: float, + min_time_limit: float = 10, ): - timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT) + 1 + timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT), time_limit) + 1 # shared memory objects times = Value("d", -1) manager = Manager() From e9feaca33805841a014fe95b43d1f8166c8cb378 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 29 Jul 2024 22:42:00 +0800 Subject: [PATCH 100/325] fix: update doc --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cf40c4d..fcbedf5 100755 --- a/README.md +++ b/README.md @@ -235,9 +235,9 @@ You are strongly recommended to use a sandbox such as [docker](https://docs.dock ```bash # Mount the current directory to the container -# If you want to change the RAM address space limit (in MB, 128 GB by default): `--max-as-limit XXX` -# If you want to change the RAM data segment limit (in MB, 4 GB by default): `--max-data-limit` -# If you want to change the RAM stack limit (in MB, 4 MB by default): `--max-stack-limit` +# If you want to change the RAM address space limit (in MB, 30 GB by default): `--max-as-limit XXX` +# If you want to change the RAM data segment limit (in MB, 30 GB by default): `--max-data-limit` +# If you want to change the RAM stack limit (in MB, 10 MB by default): `--max-stack-limit` docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl # If you only want to check the ground truths @@ -259,6 +259,8 @@ Then, run the evaluation: bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl # ...If you really don't want to check the ground truths bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt +# If you want to save the pass rate to a file +bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --save_pass_rate # You are strongly recommended to use the following command to clean up the environment after evaluation: pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; From 8d5df87b3173de33e8159b8c50e993e82780f2f4 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 29 Jul 2024 22:46:59 +0800 Subject: [PATCH 101/325] fix: update doc --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fcbedf5..16fd4a9 100755 --- a/README.md +++ b/README.md @@ -238,6 +238,7 @@ You are strongly recommended to use a sandbox such as [docker](https://docs.dock # If you want to change the RAM address space limit (in MB, 30 GB by default): `--max-as-limit XXX` # If you want to change the RAM data segment limit (in MB, 30 GB by default): `--max-data-limit` # If you want to change the RAM stack limit (in MB, 10 MB by default): `--max-stack-limit` +# If you want to increase the execution time limit (in seconds, 240 seconds by default): `--min-time-limit` docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl # If you only want to check the ground truths From bcc522af65e2692642fb31cdeb61f71b34dfb022 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 29 Jul 2024 22:53:24 +0800 Subject: [PATCH 102/325] add gradio docker releaes --- release_docker.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release_docker.sh b/release_docker.sh index 3c26ed1..3b9d104 100755 --- a/release_docker.sh +++ b/release_docker.sh @@ -28,4 +28,9 @@ docker push bigcodebench/bigcodebench-evaluate:latest docker build -f Docker/Generate.Dockerfile . -t bigcodebench/bigcodebench-generate:$version docker tag bigcodebench/bigcodebench-generate:$version bigcodebench/bigcodebench-generate:latest docker push bigcodebench/bigcodebench-generate:$version -docker push bigcodebench/bigcodebench-generate:latest \ No newline at end of file +docker push bigcodebench/bigcodebench-generate:latest + +docker build -f Docker/Gradio.Dockerfile . -t bigcodebench/bigcodebench-gradio:$version +docker tag bigcodebench/bigcodebench-gradio:$version bigcodebench/bigcodebench-gradio:latest +docker push bigcodebench/bigcodebench-gradio:$version +docker push bigcodebench/bigcodebench-gradio:latest \ No newline at end of file From 6762112a336ac01ca378927e44e25eaaccc3001e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 30 Jul 2024 02:00:22 +0800 Subject: [PATCH 103/325] fix: update time limit type conflict --- bigcodebench/eval/__init__.py | 4 ++-- bigcodebench/gen/util/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bigcodebench/eval/__init__.py b/bigcodebench/eval/__init__.py index 1a8fcfc..3596f53 100644 --- a/bigcodebench/eval/__init__.py +++ b/bigcodebench/eval/__init__.py @@ -178,8 +178,8 @@ def untrusted_check( min_time_limit: float = 10, gt_time_limit: float = 60 ) -> Tuple[str, np.ndarray]: - time_limit = max(min_time_limit, gt_time_limit) - timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT), time_limit) + 1 + min_time_limit = max(min_time_limit, gt_time_limit) + timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT), min_time_limit) + 1 # shared memory objects stat = Value("i", _UNKNOWN) manager = Manager() diff --git a/bigcodebench/gen/util/__init__.py b/bigcodebench/gen/util/__init__.py index 6e841bb..d8088ad 100644 --- a/bigcodebench/gen/util/__init__.py +++ b/bigcodebench/gen/util/__init__.py @@ -87,7 +87,7 @@ def trusted_check( max_stack_limit: float, min_time_limit: float = 10, ): - timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT), time_limit) + 1 + timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT_LIMIT), min_time_limit) + 1 # shared memory objects times = Value("d", -1) manager = Manager() From 21af8362923b740c2b75c718ef67176e1cd18ada Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 30 Jul 2024 04:14:53 +0800 Subject: [PATCH 104/325] fix: print out failed tasks --- bigcodebench/evaluate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 4500d3f..61e2a43 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -150,10 +150,11 @@ def evaluate(flags): cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") else: cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") - return if len(failed_tasks) > 0: cprint(f"Failed tasks: {failed_tasks}", "red") + + return results = { "date": datetime.now().strftime("%Y-%m-%d %H:%M"), From aa31b60af5af71e65f1e1eeecb6e9f9f7e5c285e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 21:25:02 +0800 Subject: [PATCH 105/325] fix: kill all created pids --- bigcodebench/eval/utils.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index 844d2ea..c65b849 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -221,7 +221,19 @@ def safe_exec(*args, **kwargs): try: yield finally: - # Restore original functions after the block + for pid in child_pids: + try: + os.kill(pid, signal.SIGTERM) + os.waitpid(pid, 0) + except ProcessLookupError: + pass # Process already terminated + except Exception as e: + print(f"Error terminating process {pid}: {e}") + try: + os.kill(pid, signal.SIGKILL) + except Exception: + pass + os.kill = original_kill os.killpg = original_killpg os.system = original_system From 18e9401daf54d890e3e2a6c70129c7f42c9b45b7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 21:38:58 +0800 Subject: [PATCH 106/325] fix: avoid no found pids --- bigcodebench/eval/utils.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index c65b849..25db40a 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -141,7 +141,7 @@ def safe_kill(pid, sig): else: print(f"Prevented attempt to kill PID {pid} with signal {sig}") except ProcessLookupError: - print(f"Process {pid} does not exist.") + pass def safe_killpg(pgid, sig): if pgid == current_pgid or pgid in {os.getpgid(pid) for pid in child_pids}: @@ -224,15 +224,18 @@ def safe_exec(*args, **kwargs): for pid in child_pids: try: os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) + # Wait for a short time to see if the process terminates + for _ in range(10): # Wait up to 1 second + time.sleep(0.1) + if os.waitpid(pid, os.WNOHANG) != (0, 0): + break + else: + # If the process didn't terminate, try SIGKILL + os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass # Process already terminated except Exception as e: - print(f"Error terminating process {pid}: {e}") - try: - os.kill(pid, signal.SIGKILL) - except Exception: - pass + print(f"Error handling process {pid}: {e}") os.kill = original_kill os.killpg = original_killpg From 50d1fd1839f54318287d2d9cdfc31d59afade434 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 21:43:07 +0800 Subject: [PATCH 107/325] fix: add time --- bigcodebench/eval/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index 25db40a..f1301ea 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -29,6 +29,7 @@ import tempfile import subprocess import multiprocessing +import time from typing import Optional TIMEOUT_LIMIT=240.0 From 092020000721852560e399c4fd98134c32e33b89 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 21:46:16 +0800 Subject: [PATCH 108/325] fix: avoid no child pid kill --- bigcodebench/eval/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index f1301ea..214f141 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -228,8 +228,11 @@ def safe_exec(*args, **kwargs): # Wait for a short time to see if the process terminates for _ in range(10): # Wait up to 1 second time.sleep(0.1) - if os.waitpid(pid, os.WNOHANG) != (0, 0): - break + try: + # Check if the process has terminated + os.kill(pid, 0) + except ProcessLookupError: + break # Process has terminated else: # If the process didn't terminate, try SIGKILL os.kill(pid, signal.SIGKILL) From f771678591c2ee4b1f824a165179efb0140339c9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 21:53:56 +0800 Subject: [PATCH 109/325] fix: avoid no child pid kill --- bigcodebench/eval/utils.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index 214f141..82b8085 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -225,19 +225,16 @@ def safe_exec(*args, **kwargs): for pid in child_pids: try: os.kill(pid, signal.SIGTERM) - # Wait for a short time to see if the process terminates - for _ in range(10): # Wait up to 1 second + for _ in range(10): time.sleep(0.1) try: - # Check if the process has terminated os.kill(pid, 0) except ProcessLookupError: - break # Process has terminated + break else: - # If the process didn't terminate, try SIGKILL os.kill(pid, signal.SIGKILL) except ProcessLookupError: - pass # Process already terminated + pass except Exception as e: print(f"Error handling process {pid}: {e}") From 021aee2ac55682454428967d4c925dae42e81a57 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 22:12:28 +0800 Subject: [PATCH 110/325] fix(inspect): update inspect --- bigcodebench/inspect.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bigcodebench/inspect.py b/bigcodebench/inspect.py index b06f5bd..920902e 100755 --- a/bigcodebench/inspect.py +++ b/bigcodebench/inspect.py @@ -19,7 +19,7 @@ def inspection(args): shutil.rmtree(path, ignore_errors=True) if not os.path.exists(path): os.makedirs(path) - problems = get_bigcodebench() + problems = get_bigcodebench(subset=flags.subset) eval_results = json.load(open(args.eval_results, "r")) for task_id, results in eval_results["eval"].items(): @@ -30,7 +30,7 @@ def inspection(args): os.makedirs(task_path) task_id_data = problems[task_id] with open(os.path.join(task_path, "ground_truth.py"), "w") as f: - f.write(task_id_data[f"{args.subset}_prompt"] + "\n\n" + task_id_data["canonical_solution"]) + f.write(task_id_data[f"{args.split}_prompt"] + "\n\n" + task_id_data["canonical_solution"]) # write test with open(os.path.join(task_path, "test_case.py"), "w") as f: @@ -49,7 +49,10 @@ def inspection(args): def main(): parser = argparse.ArgumentParser() parser.add_argument("--eval-results", required=True, type=str) - parser.add_argument("--subset", required=True, type=str) + parser.add_argument( + "--split", required=True, type=str, choices=["complete", "instruct"] + ) + parser.add_argument("--subset", default="hard", type=str, choices=["full", "hard"]) parser.add_argument("--in-place", action="store_true") args = parser.parse_args() From 957ea7f2da13dbd10ccad66893c4a04c896e8d78 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 22:13:48 +0800 Subject: [PATCH 111/325] fix(inspect): change flg --- bigcodebench/inspect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/inspect.py b/bigcodebench/inspect.py index 920902e..ce6fb3b 100755 --- a/bigcodebench/inspect.py +++ b/bigcodebench/inspect.py @@ -19,7 +19,7 @@ def inspection(args): shutil.rmtree(path, ignore_errors=True) if not os.path.exists(path): os.makedirs(path) - problems = get_bigcodebench(subset=flags.subset) + problems = get_bigcodebench(subset=args.subset) eval_results = json.load(open(args.eval_results, "r")) for task_id, results in eval_results["eval"].items(): From d35fd70ec1d99534364ea411c47b934435ec9558 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 22:15:58 +0800 Subject: [PATCH 112/325] fix(inspect): update args --- bigcodebench/inspect.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bigcodebench/inspect.py b/bigcodebench/inspect.py index ce6fb3b..da04fad 100755 --- a/bigcodebench/inspect.py +++ b/bigcodebench/inspect.py @@ -14,7 +14,7 @@ def inspection(args): -- completion.py: prompt + completion -- execution_trace.txt: execution trace """ - path = os.path.join("inspect", args.eval_results.split("/")[-1].replace(".json", "")) + path = os.path.join(args.save_path, args.eval_results.split("/")[-1].replace(".json", "")) if args.in_place: shutil.rmtree(path, ignore_errors=True) if not os.path.exists(path): @@ -48,12 +48,13 @@ def inspection(args): f.write("="*50 + "\n") def main(): parser = argparse.ArgumentParser() - parser.add_argument("--eval-results", required=True, type=str) + parser.add_argument("--eval_results", required=True, type=str) parser.add_argument( "--split", required=True, type=str, choices=["complete", "instruct"] ) parser.add_argument("--subset", default="hard", type=str, choices=["full", "hard"]) - parser.add_argument("--in-place", action="store_true") + parser.add_argument("--save_path", default="inspect", type=str) + parser.add_argument("--in_place", action="store_true") args = parser.parse_args() inspection(args) From a9cc5b147f65ee19685a036f1cad1dbdfd6a0620 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 22:17:48 +0800 Subject: [PATCH 113/325] fix(doc): update inspect doc --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 16fd4a9..17d066c 100755 --- a/README.md +++ b/README.md @@ -320,7 +320,11 @@ Here are some tips to speed up the evaluation: You can inspect the failed samples by using the following command: ```bash -bigcodebench.inspect --eval-results sample-sanitized-calibrated_eval_results.json --in-place +# Inspect the failed samples and save the results to `inspect/` +bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard + +# Re-run the inspection in place +bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard --in_place ``` ## 🚀 Full Script From 4ab7c7f83cc3aa4ac28752c35434ce1439c91fa0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 22:25:46 +0800 Subject: [PATCH 114/325] fix(inspect): skip problems --- bigcodebench/inspect.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigcodebench/inspect.py b/bigcodebench/inspect.py index da04fad..272166c 100755 --- a/bigcodebench/inspect.py +++ b/bigcodebench/inspect.py @@ -28,6 +28,8 @@ def inspection(args): task_path = os.path.join(path, task_id) if not os.path.exists(task_path): os.makedirs(task_path) + if task_id not in problems: + continue task_id_data = problems[task_id] with open(os.path.join(task_path, "ground_truth.py"), "w") as f: f.write(task_id_data[f"{args.split}_prompt"] + "\n\n" + task_id_data["canonical_solution"]) From 617b5bdfc96de98bc77bf82cd521310522ad6e12 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 22:28:30 +0800 Subject: [PATCH 115/325] fix(inspect): avoid empty folder --- bigcodebench/inspect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/inspect.py b/bigcodebench/inspect.py index 272166c..50c7e01 100755 --- a/bigcodebench/inspect.py +++ b/bigcodebench/inspect.py @@ -23,13 +23,13 @@ def inspection(args): eval_results = json.load(open(args.eval_results, "r")) for task_id, results in eval_results["eval"].items(): + if task_id not in problems: + continue if all(result["status"] == "pass" for result in results): continue task_path = os.path.join(path, task_id) if not os.path.exists(task_path): os.makedirs(task_path) - if task_id not in problems: - continue task_id_data = problems[task_id] with open(os.path.join(task_path, "ground_truth.py"), "w") as f: f.write(task_id_data[f"{args.split}_prompt"] + "\n\n" + task_id_data["canonical_solution"]) From 1289d538b86ec43a05f2f9286f091b3b15038479 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 2 Aug 2024 22:38:23 +0800 Subject: [PATCH 116/325] fix(docker): add extra gradio dev requires --- Docker/Gradio.Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 9d56157..0c7a0c8 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -24,6 +24,14 @@ RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench RUN pip install -I --timeout 2000 -r https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0/requirements.txt +RUN apt-get update && \ + apt-get install -y \ + bash \ + git git-lfs \ + wget curl procps \ + htop vim nano && \ + rm -rf /var/lib/apt/lists/* + WORKDIR /app RUN chown -R bigcodebenchuser:bigcodebenchuser /app From c4cc9d042d14ed2e58de44ee9d05fe803880b80d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 3 Aug 2024 01:58:54 +0800 Subject: [PATCH 117/325] update docker --- Docker/Gradio.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 9d56157..cad76e7 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -24,6 +24,8 @@ RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench RUN pip install -I --timeout 2000 -r https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0/requirements.txt +RUN pip install APScheduler==3.10.1 + WORKDIR /app RUN chown -R bigcodebenchuser:bigcodebenchuser /app From 932ac443b1e4e967f373a61ed6aabddc1213c76f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 7 Aug 2024 03:46:59 +0800 Subject: [PATCH 118/325] fix: remove printout --- bigcodebench/eval/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bigcodebench/eval/utils.py b/bigcodebench/eval/utils.py index 82b8085..6d34de9 100644 --- a/bigcodebench/eval/utils.py +++ b/bigcodebench/eval/utils.py @@ -137,7 +137,6 @@ def safe_kill(pid, sig): try: pgid = os.getpgid(pid) if pid == current_pid or pid in child_pids: - print(f"Allowed to kill PID {pid} with signal {sig}") original_kill(pid, sig) else: print(f"Prevented attempt to kill PID {pid} with signal {sig}") @@ -146,7 +145,6 @@ def safe_kill(pid, sig): def safe_killpg(pgid, sig): if pgid == current_pgid or pgid in {os.getpgid(pid) for pid in child_pids}: - print(f"Allowed to kill PGID {pgid} with signal {sig}") original_killpg(pgid, sig) else: print(f"Prevented attempt to kill PGID {pgid} with signal {sig}") From 8f6f700e0b28b0058ff6bdf2c0027499b35f9051 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 8 Aug 2024 00:36:49 +0800 Subject: [PATCH 119/325] update requires for gradio docker --- Docker/Gradio.Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 753976e..403b3d6 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -7,7 +7,6 @@ RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-b # upgrade to latest pip RUN pip install --upgrade pip -RUN pip install gradio==4.31.0 gradio[oauth] # Add a new user "bigcodebenchuser" RUN adduser --disabled-password --gecos "" bigcodebenchuser @@ -32,7 +31,7 @@ RUN apt-get update && \ htop vim nano && \ rm -rf /var/lib/apt/lists/* -RUN pip install APScheduler==3.10.1 +RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 datasets==2.14.5 huggingface-hub>=0.18.0 matplotlib==3.8.4 numpy==1.26.0 pandas==2.2.2 plotly==5.14.1 python-dateutil==2.8.2 sentencepiece tqdm==4.65.0 transformers==4.41.1 tokenizers>=0.15.0 gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio==4.31.0 gradio[oauth] gradio_leaderboard==0.0.11 requests==2.31.0 requests-oauthlib== 1.3.1 schedule == 1.2.2 WORKDIR /app From f83ba2f4857ff5db07481cb2706acb5cc6b98d54 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 8 Aug 2024 00:55:47 +0800 Subject: [PATCH 120/325] update requires for gradio docker --- Docker/Gradio.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 403b3d6..c20eda4 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -31,7 +31,7 @@ RUN apt-get update && \ htop vim nano && \ rm -rf /var/lib/apt/lists/* -RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 datasets==2.14.5 huggingface-hub>=0.18.0 matplotlib==3.8.4 numpy==1.26.0 pandas==2.2.2 plotly==5.14.1 python-dateutil==2.8.2 sentencepiece tqdm==4.65.0 transformers==4.41.1 tokenizers>=0.15.0 gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio==4.31.0 gradio[oauth] gradio_leaderboard==0.0.11 requests==2.31.0 requests-oauthlib== 1.3.1 schedule == 1.2.2 +RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 datasets==2.14.5 huggingface-hub>=0.18.0 matplotlib==3.8.4 numpy==1.26.0 pandas==2.2.2 plotly==5.14.1 python-dateutil==2.8.2 sentencepiece tqdm==4.65.0 transformers==4.41.1 tokenizers>=0.15.0 gradio-space-ci@git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio[oauth]==4.31.0 gradio_leaderboard==0.0.11 requests==2.31.0 requests-oauthlib== 1.3.1 schedule == 1.2.2 WORKDIR /app From 76669fbaf68beea84473d2070d696f9f829f38e7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 8 Aug 2024 19:16:28 +0800 Subject: [PATCH 121/325] feat: add task_fix tools --- tools/fix_v019.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tools/fix_v019.py diff --git a/tools/fix_v019.py b/tools/fix_v019.py new file mode 100644 index 0000000..21fe9f5 --- /dev/null +++ b/tools/fix_v019.py @@ -0,0 +1,56 @@ +from datasets import load_dataset, Dataset, DatasetDict +from huggingface_hub import HfApi + +import json +import copy + +BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_VERSION = "v0.1.0_hf" +BIGCODEBENCH_UPDATE = "bigcode/bcb_update" +BIGCODEBENCH_NEW_VERSION = "v0.1.1" + +def map_ds(sample): + if sample["task_id"] in ["BigCodeBench/1005", "BigCodeBench/1006"]: + sample["test"] = sample["test"].replace( + "https://getsamplefiles.com/download/zip/", + "https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0_hf/" + ) + + if sample["task_id"] in ["BigCodeBench/760"]: + for k in sample.keys(): + if "prompt" in k: + sample[k] = sample[k].replace( + "from datetime import datetime", + "import datetime" + ) + + return sample + +if __name__ == "__main__": + api = HfApi() + ds_dict = load_dataset(BIGCODEBENCH_HF) + ds = ds_dict[BIGCODEBENCH_VERSION] + function_id = [760, 1005, 1006] + + new_ds = ds.map(map_ds) + new_ds.to_json("new_ds.jsonl") + ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds + + for i in function_id: + old_sample = ds.select([i]) + new_sample = new_ds.select([i]) + old_sample.to_json("old.jsonl") + new_sample.to_json("new.jsonl") + api.upload_file( + path_or_fileobj="old.jsonl", + path_in_repo=f"{i}/old.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) + api.upload_file( + path_or_fileobj="new.jsonl", + path_in_repo=f"{i}/new.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) + From a2d20a864143362034ce844f89c1536ea1f66fda Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 8 Aug 2024 19:26:31 +0800 Subject: [PATCH 122/325] feat: add more task fix --- tools/fix_v019.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/fix_v019.py b/tools/fix_v019.py index 21fe9f5..f35e6e1 100644 --- a/tools/fix_v019.py +++ b/tools/fix_v019.py @@ -23,6 +23,17 @@ def map_ds(sample): "from datetime import datetime", "import datetime" ) + + if sample["task_id"] in ["BigCodeBench/178"]: + for k in sample.keys(): + sample[k] = sample[k].replace( + "from urllib import request\n", + "" + ) + sample[k] = sample[k].replace( + " - urllib.request\n", + "" + ) return sample @@ -30,7 +41,7 @@ def map_ds(sample): api = HfApi() ds_dict = load_dataset(BIGCODEBENCH_HF) ds = ds_dict[BIGCODEBENCH_VERSION] - function_id = [760, 1005, 1006] + function_id = [178, 760, 1005, 1006] new_ds = ds.map(map_ds) new_ds.to_json("new_ds.jsonl") From 14205d7f04007a47ce4c0150cc15cc0117f4435a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 8 Aug 2024 22:26:30 +0800 Subject: [PATCH 123/325] fix: update bbcb version --- bigcodebench/data/bigcodebench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index b113944..979ad54 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -14,7 +14,7 @@ BIGCODEBENCH_OVERRIDE_PATH = os.environ.get("BIGCODEBENCH_OVERRIDE_PATH", None) BIGCODEBENCH_HF = "bigcode/bigcodebench" -BIGCODEBENCH_VERSION = "v0.1.0_hf" +BIGCODEBENCH_VERSION = "v0.1.1" def _ready_bigcodebench_path(subset="full", version="default") -> str: if BIGCODEBENCH_OVERRIDE_PATH: From 8c3e9fa60b65c2f3acb4f0dcf62874d72f16ba8d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 8 Aug 2024 22:30:26 +0800 Subject: [PATCH 124/325] fix:update gradio docker requires --- Docker/Gradio.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index c20eda4..dee740e 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -7,6 +7,8 @@ RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-b # upgrade to latest pip RUN pip install --upgrade pip +RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 datasets==2.14.5 huggingface-hub>=0.18.0 plotly python-dateutil==2.8.2 sentencepiece tqdm==4.65.0 gradio-space-ci@git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio[oauth]==4.31.0 gradio_leaderboard==0.0.11 schedule==1.2.2 + # Add a new user "bigcodebenchuser" RUN adduser --disabled-password --gecos "" bigcodebenchuser @@ -31,7 +33,6 @@ RUN apt-get update && \ htop vim nano && \ rm -rf /var/lib/apt/lists/* -RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 datasets==2.14.5 huggingface-hub>=0.18.0 matplotlib==3.8.4 numpy==1.26.0 pandas==2.2.2 plotly==5.14.1 python-dateutil==2.8.2 sentencepiece tqdm==4.65.0 transformers==4.41.1 tokenizers>=0.15.0 gradio-space-ci@git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio[oauth]==4.31.0 gradio_leaderboard==0.0.11 requests==2.31.0 requests-oauthlib== 1.3.1 schedule == 1.2.2 WORKDIR /app From cd1feff9460aba44ca1e07742670b0f1fb1971ae Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 9 Aug 2024 19:48:33 +0800 Subject: [PATCH 125/325] remove 1005 fix --- tools/fix_v019.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/tools/fix_v019.py b/tools/fix_v019.py index f35e6e1..6476c87 100644 --- a/tools/fix_v019.py +++ b/tools/fix_v019.py @@ -5,15 +5,34 @@ import copy BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_HARD_HF = "bigcode/bigcodebench-hard" BIGCODEBENCH_VERSION = "v0.1.0_hf" BIGCODEBENCH_UPDATE = "bigcode/bcb_update" BIGCODEBENCH_NEW_VERSION = "v0.1.1" def map_ds(sample): - if sample["task_id"] in ["BigCodeBench/1005", "BigCodeBench/1006"]: + + if sample["task_id"] in ["BigCodeBench/1006"]: sample["test"] = sample["test"].replace( - "https://getsamplefiles.com/download/zip/", - "https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0_hf/" +'''\ + def test_valid_zip_url(self): + """Test a valid ZIP URL.""" + url = "https://getsamplefiles.com/download/zip/sample-1.zip" + result = task_func(url) + self.assertTrue(result.startswith("mnt/data/downloads/")) + self.assertTrue(result.endswith("sample-1")) + shutil.rmtree("mnt/data/downloads") +''', +'''\ + @patch("requests.get") + def test_non_zip_content(self, mock_get): + """Test a valid ZIP URL.""" + mock_get.return_value.status_code = 200 + mock_get.return_value.headers = {"Content-Type": "application/zip"} + mock_get.return_value.content = b"1" + url = "https://valid-url.com/sample.zip" + result = task_func(url) +''', ) if sample["task_id"] in ["BigCodeBench/760"]: @@ -40,12 +59,20 @@ def map_ds(sample): if __name__ == "__main__": api = HfApi() ds_dict = load_dataset(BIGCODEBENCH_HF) + hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) ds = ds_dict[BIGCODEBENCH_VERSION] - function_id = [178, 760, 1005, 1006] + hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] + function_id = [178, 760, 1006] new_ds = ds.map(map_ds) - new_ds.to_json("new_ds.jsonl") + new_ds.to_json("BigCodeBench.jsonl") ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds + ds_dict.push_to_hub(BIGCODEBENCH_HF) + + new_hard_ds = hard_ds.map(map_ds) + new_hard_ds.to_json("BigCodeBench-Hard.jsonl") + hard_ds_dict[BIGCODEBENCH_NEW_VERSION] = new_hard_ds + hard_ds_dict.push_to_hub(BIGCODEBENCH_HARD_HF) for i in function_id: old_sample = ds.select([i]) From 19baeb475f379b535932255a71b3578cda2d04a5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 11 Aug 2024 03:52:47 +0800 Subject: [PATCH 126/325] fix(codegen): backoff gemini rate limit --- bigcodebench/model.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/bigcodebench/model.py b/bigcodebench/model.py index 7dd77b7..fa9ad34 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -458,20 +458,24 @@ def codegen( outputs = [] for _ in range(batch_size): - response = model.generate_content( - "Please generate self-contained code to complete the following problem wrapped in a Python markdown block:" - + f"\n```python\n{prompt.strip()}\n```", - generation_config=genai_config - ) - try: - output = response.candidates[0].content.parts[0].text - outputs.append(output) - except Exception as e: - if "list index out of range" in str(e): - # append dummy response - outputs.append("NO_RESPONSE") - else: - raise e + while True: + try: + response = model.generate_content( + "Please generate self-contained code to complete the following problem wrapped in a Python markdown block:" + + f"\n```python\n{prompt.strip()}\n```", + generation_config=genai_config + ) + output = response.candidates[0].content.parts[0].text + outputs.append(output) + break + except Exception as e: + if "list index out of range" in str(e): + # append dummy response + outputs.append("NO_RESPONSE") + break + else: + print(e) + continue return outputs From 10d062f4ce907c7cf035accc018c2c42d306bcb1 Mon Sep 17 00:00:00 2001 From: Sanjay Krishna Gouda Date: Thu, 15 Aug 2024 00:28:42 -0400 Subject: [PATCH 127/325] add multiprocessing support for sanitization step --- README.md | 3 ++ Requirements/requirements.txt | 3 +- bigcodebench/sanitize.py | 95 +++++++++++++++++++++++------------ 3 files changed, 67 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 17d066c..10922e5 100755 --- a/README.md +++ b/README.md @@ -193,6 +193,9 @@ We provide a tool namely `bigcodebench.sanitize` to clean up the code: bigcodebench.sanitize --samples samples.jsonl --calibrate # Sanitized code will be produced to `samples-sanitized-calibrated.jsonl` +# 💡 Optionally run the sanitization step with multiprocessing to speedup +bigcodebench.sanitize --samples samples.jsonl --calibrate --parallel 8 + # 💡 If you want to get the original results: bigcodebench.sanitize --samples samples.jsonl # Sanitized code will be produced to `samples-sanitized.jsonl` diff --git a/Requirements/requirements.txt b/Requirements/requirements.txt index 69d8b6c..178ae81 100644 --- a/Requirements/requirements.txt +++ b/Requirements/requirements.txt @@ -1,9 +1,10 @@ appdirs>=1.4.4 fire>=0.6.0 multipledispatch>=0.6.0 +pqdm>=0.2.0 tempdir>=0.7.1 termcolor>=2.0.0 tqdm>=4.56.0 tree_sitter_languages>=1.10.2 tree-sitter==0.21.3 -wget>=3.2 \ No newline at end of file +wget>=3.2 diff --git a/bigcodebench/sanitize.py b/bigcodebench/sanitize.py index 6a93f2e..df9ed4e 100644 --- a/bigcodebench/sanitize.py +++ b/bigcodebench/sanitize.py @@ -3,6 +3,7 @@ import os import pathlib from typing import Dict, Generator, List, Optional, Set, Tuple +from pqdm.processes import pqdm from tqdm import tqdm from tree_sitter import Node @@ -178,8 +179,48 @@ def sanitize(code: str, entrypoint: Optional[str] = None) -> str: return sanitized_output +def process_solution( + sample_solution: Dict, + dataset: Dict, + entry_point: Dict, + debug_task: str = None, + calibrate: bool = False, + is_folder: bool = False, + target_path: str = None, +): + + task_id = sample_solution.get("task_id") + if not task_id or task_id not in dataset: + return None + + dbg_identifier = sample_solution["_identifier"] + if debug_task is not None and task_id != debug_task: + return None + + function_name = entry_point.get(task_id) + old_code = sample_solution.get("solution") + + if old_code is None: + assert "completion" in sample_solution, sample_solution + old_code = dataset[task_id]["complete_prompt"] + "\n" + sample_solution.get("completion") + else: + if calibrate: + old_code = old_code.replace("```python\n ", "```python\n"+dataset[task_id]["complete_prompt"]+" ") + + new_code = sanitize(code=old_code, entrypoint=function_name) + + # if old code and new code are different, print msg + if new_code != old_code: + msg = "Sanitized: " + dbg_identifier + if is_folder: + msg += " -> " + dbg_identifier.replace(samples, target_path) + print(msg) + + return {"task_id": task_id, "solution": new_code} + + def script( - samples: str, inplace: bool = False, debug_task: str = None, calibrate: bool = False + samples: str, inplace: bool = False, debug_task: str = None, calibrate: bool = False, parallel: int=32 ): # task_id -> entry_point entry_point = {} @@ -211,38 +252,26 @@ def script( new_solutions = [] - for solution in tqdm(load_solutions(samples)): - task_id = solution["task_id"] - if task_id not in dataset: - print( - f"Skiping {task_id} as it does not existing in the latest EvalPlus dataset." - ) - continue - - function_name = entry_point[task_id] if task_id in entry_point else None - dbg_identifier = solution["_identifier"] - if debug_task is not None and task_id != debug_task: - continue - - ntotal += 1 - if "solution" in solution: - old_code = solution["solution"] - if calibrate: - old_code = solution["solution"].replace("```python\n ", "```python\n"+dataset[task_id]["complete_prompt"]+" ") - else: - assert "completion" in solution - old_code = dataset[task_id]["complete_prompt"] + "\n" + solution["completion"] - - new_code = sanitize(code=old_code, entrypoint=function_name) - # if changed, print the message - if new_code != old_code: - msg = "Sanitized: " + dbg_identifier - if is_folder: - msg += " -> " + dbg_identifier.replace(samples, target_path) - print(msg) + parallel_arg_list = [ + { + "sample_solution": sample_solution, + "dataset": dataset, + "entry_point": entry_point, + "debug_task": debug_task, + "calibrate": calibrate, + "is_folder": is_folder, + "target_path": target_path + } + for sample_solution in load_solutions(samples) + ] + + results = pqdm(parallel_arg_list, process_solution, n_jobs=min(parallel, os.cpu_count()), argument_type="kwargs") + + for result in results: + if result is not None: + new_solutions.append(result) nsan += 1 - - new_solutions.append({"task_id": task_id, "solution": new_code}) + ntotal += 1 if is_folder: write_directory(target_path, new_solutions) @@ -263,4 +292,4 @@ def main(): if __name__ == "__main__": - main() + main() \ No newline at end of file From c90f3d3af798154812dcc2a263715293a3d8f14e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 21 Aug 2024 04:38:01 +0800 Subject: [PATCH 128/325] fix(gradio-docker): update dependencies --- Docker/Gradio.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index dee740e..9de820b 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-b # upgrade to latest pip RUN pip install --upgrade pip -RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 datasets==2.14.5 huggingface-hub>=0.18.0 plotly python-dateutil==2.8.2 sentencepiece tqdm==4.65.0 gradio-space-ci@git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio[oauth]==4.31.0 gradio_leaderboard==0.0.11 schedule==1.2.2 +RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 huggingface-hub>=0.18.0 plotly python-dateutil==2.8.2 gradio-space-ci@git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio[oauth]==4.31.0 gradio_leaderboard==0.0.11 schedule==1.2.2 # Add a new user "bigcodebenchuser" RUN adduser --disabled-password --gecos "" bigcodebenchuser From bad8ad2fa773a25933dab8c864358203343c38ee Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 21 Aug 2024 04:38:57 +0800 Subject: [PATCH 129/325] fix: add pqdm --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 6f1c731..176759b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ install_requires = appdirs>=1.4.4 fire>=0.6.0 multipledispatch>=0.6.0 + pqdm>=0.2.0 tempdir>=0.7.1 termcolor>=2.0.0 tqdm>=4.56.0 From 2f431fa7af597c4f301c3875583ab18ecb7e1b1b Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 20 Aug 2024 21:15:02 +0000 Subject: [PATCH 130/325] add phi-3.5-mini-instruct --- analysis/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index 88fc003..5a0b306 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -774,7 +774,7 @@ "open-data": "Partial", }, "new-microsoft/Phi-3-mini-128k-instruct": { - "name": "Phi-3-Mini-128K-Instruct (June 2024)", + "name": "Phi-3.1-Mini-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct", "prompted": True, "moe": False, @@ -783,7 +783,7 @@ "open-data": "None", }, "old-microsoft/Phi-3-mini-128k-instruct": { - "name": "Phi-3-Mini-128K-Instruct (Old)", + "name": "Phi-3-Mini-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct", "prompted": True, "moe": False, @@ -971,4 +971,13 @@ "act_param": 21, "open-data": "None", }, + "microsoft/Phi-3.5-mini-instruct": { + "name": "Phi-3.5-Mini-Instruct", + "link": "https://huggingface.co/microsoft/Phi-3.5-mini-instruct", + "prompted": True, + "moe": False, + "size": 3.8, + "act_param": 3.8, + "open-data": "None", + }, } \ No newline at end of file From b92b340b95de3b78b526f6c7699a2621b9945eec Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 22 Aug 2024 05:33:00 +0800 Subject: [PATCH 131/325] fix model list --- analysis/utils.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index 5a0b306..a0393df 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -980,4 +980,58 @@ "act_param": 3.8, "open-data": "None", }, + "nv-mistralai--mistral-nemo-12b-instruct": { + "name": "Mistral-Nemo-12B-Instruct", + "link": "https://huggingface.co/nv-mistralai/Mistral-Nemo-12B-Instruct", + "prompted": True, + "moe": False, + "size": 12, + "act_param": 12, + "open-data": "None", + }, + "wyt2000/InverseCoder-CL-13B": { + "name": "InverseCoder-CL-13B", + "link": "https://huggingface.co/wyt2000/InverseCoder-CL-13B", + "prompted": True, + "moe": False, + "size": 13, + "act_param": 13, + "open-data": "Partial", + }, + "wyt2000/InverseCoder-CL-7B": { + "name": "InverseCoder-CL-7B", + "link": "https://huggingface.co/wyt2000/InverseCoder-CL-7B", + "prompted": True, + "moe": False, + "size": 7, + "act_param": 7, + "open-data": "Partial", + }, + "wyt2000/InverseCoder-CL-6.7B": { + "name": "InverseCoder-CL-6.7B", + "link": "https://huggingface.co/wyt2000/InverseCoder-CL-6.7B", + "prompted": True, + "moe": False, + "size": 6.7, + "act_param": 6.7, + "open-data": "Partial", + }, + "gemini-1.5-pro-exp-0801": { + "name": "Gemini-1.5-Pro-Exp-0801", + "link": "https://deepmind.google/technologies/gemini/pro", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "gpt-4o-2024-08-06": { + "name": "GPT-4o-2024-08-06", + "link": "https://openai.com/index/introducing-structured-outputs-in-the-api/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, } \ No newline at end of file From 4cfa8569f57e3caa6b36f0be622891b82406e656 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 22 Aug 2024 09:12:01 +0800 Subject: [PATCH 132/325] fix model list --- analysis/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index a0393df..ef87958 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1007,9 +1007,9 @@ "act_param": 7, "open-data": "Partial", }, - "wyt2000/InverseCoder-CL-6.7B": { - "name": "InverseCoder-CL-6.7B", - "link": "https://huggingface.co/wyt2000/InverseCoder-CL-6.7B", + "wyt2000/InverseCoder-DS-6.7B": { + "name": "InverseCoder-DS-6.7B", + "link": "https://huggingface.co/wyt2000/InverseCoder-DS-6.7B", "prompted": True, "moe": False, "size": 6.7, From ffcbc443e18efabe5ccacfaa360178b37bb5ca1a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 25 Aug 2024 06:19:27 +0800 Subject: [PATCH 133/325] add abacusai models --- analysis/utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index ef87958..41f50b0 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1034,4 +1034,22 @@ "act_param": None, "open-data": "None", }, + "abacusai/Dracarys-Llama-3.1-70B-Instruct": { + "name": "Dracarys-Llama-3.1-70B-Instruct", + "link": "https://huggingface.co/abacusai/Dracarys-Llama-3.1-70B-Instruct", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "abacusai/Dracarys-72B-Instruct": { + "name": "Dracarys-72B-Instruct", + "link": "https://huggingface.co/abacusai/Dracarys-72B-Instruct", + "prompted": True, + "moe": False, + "size": 72, + "act_param": 72, + "open-data": "None", + }, } \ No newline at end of file From e9dea62fdc1a4bb9f85ed2fd360fe24882d4f101 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 26 Aug 2024 01:57:58 +0800 Subject: [PATCH 134/325] feat(decontamination): add decontamination script --- decontamination/n_gram_check.py | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 decontamination/n_gram_check.py diff --git a/decontamination/n_gram_check.py b/decontamination/n_gram_check.py new file mode 100644 index 0000000..01f1e58 --- /dev/null +++ b/decontamination/n_gram_check.py @@ -0,0 +1,76 @@ +from datasets import load_dataset, load_from_disk +from collections import Counter +import tiktoken +from nltk import ngrams +from tqdm import tqdm +import datasets + +def has_overlap(sample_1, sample_2): + """Check if there is any N-gram overlap between the long string and a given string.""" + return not set(sample_1).isdisjoint(set(sample_2)) + +from concurrent.futures import ThreadPoolExecutor, as_completed + +def calculate_overlap_percentage(samples_1, samples_2): + def check_sample(sample): + for long_sample in samples_2: + if has_overlap(sample, long_sample["ngram"]): + return 1 + return 0 + + count = 0 + with ThreadPoolExecutor() as executor: + futures = [executor.submit(check_sample, sample) for sample in samples_1] + for future in tqdm(as_completed(futures), total=len(futures)): + count += future.result() + + return count / len(samples_1) * 100 + +def load_odex_data(n=10): + def map_ngram(sample): + return {"ngram": set([" ".join(ngram) for ngram in ngrams(sample["intent"].split(), n)])} + dataset = load_dataset("neulab/odex", "en", split="test") + dataset = dataset.map(map_ngram, num_proc=16, batch_size=16, remove_columns=dataset.column_names) + return dataset + +def load_stackoverflow(n=10): + def map_ngram(sample): + return {"ngram": set([" ".join(ngram) for ngram in ngrams(sample["question"].split(), n)])} + dataset = load_dataset("bigcode/stack-exchange-preferences-20230914-clean-anonymization", split="train") + dataset = dataset.map(map_ngram, num_proc=16, batch_size=16, remove_columns=dataset.column_names) + dataset.push_to_hub(f"stackoverflow_ngram_{n}") + return dataset + + +def load_starcoderdata(n=10): + def map_ngram(sample): + return {"ngram": set([" ".join(ngram) for ngram in ngrams(sample["content"].split(), n)])} + dataset = load_dataset("bigcode/starcoderdata", data_dir="python", split="train") + dataset = dataset.map(map_ngram, num_proc=16, batch_size=16, remove_columns=dataset.column_names) + dataset.push_to_hub(f"starcoderdata_ngram_{n}") + return dataset + +def load_bigcodebench(n=10): + def map_ngram(sample): + return {"ngram": set([" ".join(ngram) for ngram in ngrams(sample["instruct_prompt"].split("```")[0].split(), n)])} + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_hf") + dataset = dataset.map(map_ngram, num_proc=16, batch_size=16, remove_columns=dataset.column_names) + dataset.push_to_hub(f"bigcodebench_ngram_{n}") + return dataset + + +if __name__ == "__main__": + n_gram_size = 10 + N_SHARDS = 50 + user_name = "terryyz" + bigcodebench = load_dataset(f"{user_name}/bigcodebench_ngram_{n_gram_size}", split="train") + + dataset_name = "starcoderdata" + print(dataset_name, n_gram_size) + indices = [] + for i in tqdm(range(N_SHARDS)): + ds = load_dataset(f"{user_name}/{dataset_name}_ngram_{n_gram_size}_overlap_{i}", split="train") + overlap_indices = [idx for idx, example in enumerate(ds) if example["overlap"]] + indices.extend(overlap_indices) + with open(f"{dataset_name}_ngram_{n_gram_size}_overlap.txt", "w") as f: + f.write(f"{len(set(indices))/1140*100:.2f}%") \ No newline at end of file From 31c7bfddc58f01605baca9c3d8194291d60da417 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 26 Aug 2024 01:58:44 +0800 Subject: [PATCH 135/325] feat(decontamination): add stats --- decontamination/odex_10_overlap.txt | 1 + decontamination/odex_13_overlap.txt | 1 + decontamination/stackoverflow_10_overlap.txt | 1 + decontamination/stackoverflow_13_overlap.txt | 1 + decontamination/starcoderdata_10_overlap.txt | 1 + 5 files changed, 5 insertions(+) create mode 100644 decontamination/odex_10_overlap.txt create mode 100644 decontamination/odex_13_overlap.txt create mode 100644 decontamination/stackoverflow_10_overlap.txt create mode 100644 decontamination/stackoverflow_13_overlap.txt create mode 100644 decontamination/starcoderdata_10_overlap.txt diff --git a/decontamination/odex_10_overlap.txt b/decontamination/odex_10_overlap.txt new file mode 100644 index 0000000..2a1c9b9 --- /dev/null +++ b/decontamination/odex_10_overlap.txt @@ -0,0 +1 @@ +0.09% \ No newline at end of file diff --git a/decontamination/odex_13_overlap.txt b/decontamination/odex_13_overlap.txt new file mode 100644 index 0000000..be01eee --- /dev/null +++ b/decontamination/odex_13_overlap.txt @@ -0,0 +1 @@ +odex: 0.00% \ No newline at end of file diff --git a/decontamination/stackoverflow_10_overlap.txt b/decontamination/stackoverflow_10_overlap.txt new file mode 100644 index 0000000..96202f9 --- /dev/null +++ b/decontamination/stackoverflow_10_overlap.txt @@ -0,0 +1 @@ +1.49% \ No newline at end of file diff --git a/decontamination/stackoverflow_13_overlap.txt b/decontamination/stackoverflow_13_overlap.txt new file mode 100644 index 0000000..95cbb56 --- /dev/null +++ b/decontamination/stackoverflow_13_overlap.txt @@ -0,0 +1 @@ +0.18% \ No newline at end of file diff --git a/decontamination/starcoderdata_10_overlap.txt b/decontamination/starcoderdata_10_overlap.txt new file mode 100644 index 0000000..76b24b7 --- /dev/null +++ b/decontamination/starcoderdata_10_overlap.txt @@ -0,0 +1 @@ +2.54% \ No newline at end of file From 83008e90ce515bc95e7199073f25aaa16fc7029f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 26 Aug 2024 01:59:31 +0800 Subject: [PATCH 136/325] fix: add eos as hf stop string --- bigcodebench/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigcodebench/model.py b/bigcodebench/model.py index fa9ad34..c5093b0 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -228,6 +228,8 @@ def codegen( do_sample=do_sample, num_return_sequences=min(self.batch_size, num_samples), pad_token_id=self.tokenizer.eos_token_id, + stop_strings=self.eos, + tokenizer=self.tokenizer, **kwargs, ) From 1b20d34fe8504980a3eb945918ccb9bbd10c6171 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 26 Aug 2024 02:09:54 +0800 Subject: [PATCH 137/325] fix(data): add pandas to BigCodeBench/37 --- tools/fix_v0110.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tools/fix_v0110.py diff --git a/tools/fix_v0110.py b/tools/fix_v0110.py new file mode 100644 index 0000000..d1d1300 --- /dev/null +++ b/tools/fix_v0110.py @@ -0,0 +1,59 @@ +from datasets import load_dataset, Dataset, DatasetDict +from huggingface_hub import HfApi + +import json +import copy + +BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_HARD_HF = "bigcode/bigcodebench-hard" +BIGCODEBENCH_VERSION = "v0.1.1" +BIGCODEBENCH_UPDATE = "bigcode/bcb_update" +BIGCODEBENCH_NEW_VERSION = "v0.1.2" + +def map_ds(sample): + if sample["task_id"] in ["BigCodeBench/37"]: + for k in sample.keys(): + if "prompt" in k: + sample[k] = "import pandas as pd\n" + sample[k] + sample[k] = sample[k].replace( + "Requirements:\n - sklearn.ensemble\n", + "Requirements:\n - pandas\n - sklearn.ensemble\n" + ) + + return sample + +if __name__ == "__main__": + api = HfApi() + ds_dict = load_dataset(BIGCODEBENCH_HF) + hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) + ds = ds_dict[BIGCODEBENCH_VERSION] + hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] + function_id = [37] + + new_ds = ds.map(map_ds) + new_ds.to_json("BigCodeBench.jsonl") + ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds + ds_dict.push_to_hub(BIGCODEBENCH_HF) + + new_hard_ds = hard_ds.map(map_ds) + new_hard_ds.to_json("BigCodeBench-Hard.jsonl") + hard_ds_dict[BIGCODEBENCH_NEW_VERSION] = new_hard_ds + hard_ds_dict.push_to_hub(BIGCODEBENCH_HARD_HF) + + for i in function_id: + old_sample = ds.select([i]) + new_sample = new_ds.select([i]) + old_sample.to_json("old.jsonl") + new_sample.to_json("new.jsonl") + api.upload_file( + path_or_fileobj="old.jsonl", + path_in_repo=f"{i}/old.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) + api.upload_file( + path_or_fileobj="new.jsonl", + path_in_repo=f"{i}/new.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) From 831e1854fa340c1ce1c242c28d835b88518deede Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 26 Aug 2024 05:13:15 +0800 Subject: [PATCH 138/325] fix(doc): add news --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 10922e5..2401328 100755 --- a/README.md +++ b/README.md @@ -27,6 +27,15 @@ 🙏Acknowledgement

+## News +- **[2024-08-19]** To make the evaluation fully reproducible, we add a real-time code execution session to the leaderboard. It can be viewed [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). +- **[2024-08-02]** We release `bigcodebench==v0.1.9`. +- **[2024-07-18]** We announce a subset of BigCodeBench, BigCodeBench-Hard, which includes 148 tasks that are more aligned with the real-world programming tasks. The details are available [in this blog post](https://huggingface.co/blog/terryyz/bigcodebench-hard). The dataset is available [here](https://huggingface.co/datasets/bigcode/bigcodebench-hard). The new release is `bigcodebench==v0.1.8`. +- **[2024-06-28]** We release `bigcodebench==v0.1.7`. +- **[2024-06-27]** We release `bigcodebench==v0.1.6`. +- **[2024-06-19]** We start the Hugging Face BigCodeBench Leaderboard! The leaderboard is available [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). +- **[2024-06-18]** We release BigCodeBench, a new benchmark for code generation with 1140 software-engineering-oriented programming tasks. Preprint is available [here](https://arxiv.org/abs/2406.15877). PyPI package is available [here](https://pypi.org/project/bigcodebench/) with the version `0.1.5`. + ## 🌸 About ### BigCodeBench @@ -361,9 +370,9 @@ We share pre-generated code samples from LLMs we have [evaluated](https://huggin - [ ] Due to the flakiness in the evaluation, the execution results may vary slightly (~0.2% for Full set, and ~0.6% for Hard set) between runs. We are working on improving the evaluation stability. -- [ ] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. +- [x] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. If the issue persists ,please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). -- [ ] We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code. +- [x] ~~We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code.~~ Please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). ## 📜 Citation From aa74075b40f9d818f84861acfdc41a149b7bf941 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 28 Aug 2024 03:27:24 +0800 Subject: [PATCH 139/325] fix(doc): remove comparisons --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 2401328..5bb72ba 100755 --- a/README.md +++ b/README.md @@ -50,12 +50,6 @@ BigCodeBench focuses on the evaluation of LLM4Code with *diverse function calls* * ✨ **Precise evaluation & ranking**: See [our leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard) for latest LLM rankings before & after rigorous evaluation. * ✨ **Pre-generated samples**: BigCodeBench accelerates code intelligence research by open-sourcing [LLM-generated samples](#-LLM-generated-code) for various models -- no need to re-run the expensive benchmarks! -### Main Differences from EvalPlus - -We inherit the design of the EvalPlus framework, which is a flexible and extensible evaluation framework for code generation tasks. However, BigCodeBench has the following differences: -* Execution Environment: The execution environment in BigCodeBench is less bounded than EvalPlus to support tasks with diverse library dependencies. -* Test Evaluation: BigCodeBench relies on `unittest` for evaluating the generated code, which is more suitable for the test harness in BigCodeBench. - ## 🔥 Quick Start > [!Tip] From db8200e0bb2006766223a7a3227789a8eb7e34c9 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Mon, 2 Sep 2024 00:03:49 +0200 Subject: [PATCH 140/325] Remove extra period in task BigCodeBench/16 --- tools/fix_020.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tools/fix_020.py diff --git a/tools/fix_020.py b/tools/fix_020.py new file mode 100644 index 0000000..82b48d7 --- /dev/null +++ b/tools/fix_020.py @@ -0,0 +1,59 @@ +from datasets import load_dataset, Dataset, DatasetDict +from huggingface_hub import HfApi + +import json +import copy + +BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_HARD_HF = "bigcode/bigcodebench-hard" +BIGCODEBENCH_VERSION = "v0.1.0_hf" +BIGCODEBENCH_UPDATE = "bigcode/bcb_update" +BIGCODEBENCH_NEW_VERSION = "v0.1.1" # TODO(hvaara): [DO NOT MERGE] Figure out which version we're targeting + + +def map_ds(sample): + + if sample["task_id"] in ["BigCodeBench/16"]: + for k in sample.keys(): + sample[k] = sample[k].replace( + "No logs found to backup.", "No logs found to backup" + ) + + return sample + + +if __name__ == "__main__": + api = HfApi() + ds_dict = load_dataset(BIGCODEBENCH_HF) + hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) + ds = ds_dict[BIGCODEBENCH_VERSION] + hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] + function_id = [16] + + new_ds = ds.map(map_ds) + new_ds.to_json("BigCodeBench.jsonl") + ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds + ds_dict.push_to_hub(BIGCODEBENCH_HF) + + new_hard_ds = hard_ds.map(map_ds) + new_hard_ds.to_json("BigCodeBench-Hard.jsonl") + hard_ds_dict[BIGCODEBENCH_NEW_VERSION] = new_hard_ds + hard_ds_dict.push_to_hub(BIGCODEBENCH_HARD_HF) + + for i in function_id: + old_sample = ds.select([i]) + new_sample = new_ds.select([i]) + old_sample.to_json("old.jsonl") + new_sample.to_json("new.jsonl") + api.upload_file( + path_or_fileobj="old.jsonl", + path_in_repo=f"{i}/old.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) + api.upload_file( + path_or_fileobj="new.jsonl", + path_in_repo=f"{i}/new.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) From 479957525b1240fb9bb2ee70b452da02fb2a9562 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Tue, 3 Sep 2024 23:20:04 +0200 Subject: [PATCH 141/325] Target v0.1.10 for task BigCodeBench/16 --- tools/fix_020.py | 59 ---------------------------------------------- tools/fix_v0110.py | 8 ++++++- 2 files changed, 7 insertions(+), 60 deletions(-) delete mode 100644 tools/fix_020.py diff --git a/tools/fix_020.py b/tools/fix_020.py deleted file mode 100644 index 82b48d7..0000000 --- a/tools/fix_020.py +++ /dev/null @@ -1,59 +0,0 @@ -from datasets import load_dataset, Dataset, DatasetDict -from huggingface_hub import HfApi - -import json -import copy - -BIGCODEBENCH_HF = "bigcode/bigcodebench" -BIGCODEBENCH_HARD_HF = "bigcode/bigcodebench-hard" -BIGCODEBENCH_VERSION = "v0.1.0_hf" -BIGCODEBENCH_UPDATE = "bigcode/bcb_update" -BIGCODEBENCH_NEW_VERSION = "v0.1.1" # TODO(hvaara): [DO NOT MERGE] Figure out which version we're targeting - - -def map_ds(sample): - - if sample["task_id"] in ["BigCodeBench/16"]: - for k in sample.keys(): - sample[k] = sample[k].replace( - "No logs found to backup.", "No logs found to backup" - ) - - return sample - - -if __name__ == "__main__": - api = HfApi() - ds_dict = load_dataset(BIGCODEBENCH_HF) - hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) - ds = ds_dict[BIGCODEBENCH_VERSION] - hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] - function_id = [16] - - new_ds = ds.map(map_ds) - new_ds.to_json("BigCodeBench.jsonl") - ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds - ds_dict.push_to_hub(BIGCODEBENCH_HF) - - new_hard_ds = hard_ds.map(map_ds) - new_hard_ds.to_json("BigCodeBench-Hard.jsonl") - hard_ds_dict[BIGCODEBENCH_NEW_VERSION] = new_hard_ds - hard_ds_dict.push_to_hub(BIGCODEBENCH_HARD_HF) - - for i in function_id: - old_sample = ds.select([i]) - new_sample = new_ds.select([i]) - old_sample.to_json("old.jsonl") - new_sample.to_json("new.jsonl") - api.upload_file( - path_or_fileobj="old.jsonl", - path_in_repo=f"{i}/old.jsonl", - repo_id=BIGCODEBENCH_UPDATE, - # repo_type="dataset" - ) - api.upload_file( - path_or_fileobj="new.jsonl", - path_in_repo=f"{i}/new.jsonl", - repo_id=BIGCODEBENCH_UPDATE, - # repo_type="dataset" - ) diff --git a/tools/fix_v0110.py b/tools/fix_v0110.py index d1d1300..a6aadac 100644 --- a/tools/fix_v0110.py +++ b/tools/fix_v0110.py @@ -11,6 +11,12 @@ BIGCODEBENCH_NEW_VERSION = "v0.1.2" def map_ds(sample): + if sample["task_id"] in ["BigCodeBench/16"]: + for k in sample.keys(): + sample[k] = sample[k].replace( + "No logs found to backup.", "No logs found to backup" + ) + if sample["task_id"] in ["BigCodeBench/37"]: for k in sample.keys(): if "prompt" in k: @@ -28,7 +34,7 @@ def map_ds(sample): hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) ds = ds_dict[BIGCODEBENCH_VERSION] hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] - function_id = [37] + function_id = [16, 37] new_ds = ds.map(map_ds) new_ds.to_json("BigCodeBench.jsonl") From e71fe2e4d6e12d3d603fd9f1d2cacb2b26fa95f5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 11 Sep 2024 00:40:02 +0800 Subject: [PATCH 142/325] fix (data): update 241 & 267 --- tools/fix_v0110.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/fix_v0110.py b/tools/fix_v0110.py index a6aadac..e96e014 100644 --- a/tools/fix_v0110.py +++ b/tools/fix_v0110.py @@ -26,6 +26,22 @@ def map_ds(sample): "Requirements:\n - pandas\n - sklearn.ensemble\n" ) + if sample["task_id"] in ["BigCodeBench/241"]: + for k in sample.keys(): + if "prompt" in k: + sample[k] = sample[k].replace( + "The function will plot the original and normalized arrays using matplotlib.", + "The function will plot the original and normalized arrays with a title of 'Original vs. Normalized Data'." + ) + + if sample["task_id"] in ["BigCodeBench/267"]: + for k in sample.keys(): + if "prompt" in k: + sample[k] = sample[k].replace( + "Plots and returns the FFT of the signal.", + "Plots and returns the FFT of the signal with a title of 'FFT of the signal'." + ) + return sample if __name__ == "__main__": @@ -34,7 +50,7 @@ def map_ds(sample): hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) ds = ds_dict[BIGCODEBENCH_VERSION] hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] - function_id = [16, 37] + function_id = [16, 37, 241, 267] new_ds = ds.map(map_ds) new_ds.to_json("BigCodeBench.jsonl") From ad3e0877148fc6585d0a1b36116fa383334e8ae3 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Sat, 14 Sep 2024 19:41:40 +0200 Subject: [PATCH 143/325] Increase frequency of progress checker --- bigcodebench/evaluate.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 61e2a43..7b926e2 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -204,14 +204,26 @@ def evaluate(flags): assert len(completion_id) == len(problems), "Missing problems in samples" def stucking_checker(): + unchanged_duration = 0 + last_size = len(remainings) + while remainings: - last_size = len(remainings) - time.sleep(240) - if last_size != len(remainings) or len(remainings) == 0: - continue - # Potential stucking - warn("No samples had finished testing in the last 240s") - warn(f"{len(remainings)} samples to be tested: {remainings}") + time.sleep(1) + current_size = len(remainings) + + if current_size != last_size or current_size == 0: + # Reset the unchanged duration if something has changed + unchanged_duration = 0 + last_size = current_size + else: + # Increment the duration if nothing has changed + unchanged_duration += 1 + + if unchanged_duration >= 240: + # Output warnings after 240 seconds of no change + warn("No samples have finished testing in the last 240s") + warn(f"{len(remainings)} samples to be tested: {remainings}") + unchanged_duration = 0 # Reset after warning threading.Thread(target=stucking_checker).start() From bf58e0f124365f5f916f8fb5fa931a15cbcb6868 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Sat, 14 Sep 2024 20:07:53 +0200 Subject: [PATCH 144/325] Wait on futures in progress checker --- bigcodebench/evaluate.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 7b926e2..0451d2d 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -6,7 +6,7 @@ import threading import time from collections import Counter, defaultdict -from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures import ProcessPoolExecutor, as_completed, wait, ALL_COMPLETED from datetime import datetime from typing import Any, Dict, List, Tuple from warnings import warn @@ -204,26 +204,14 @@ def evaluate(flags): assert len(completion_id) == len(problems), "Missing problems in samples" def stucking_checker(): - unchanged_duration = 0 - last_size = len(remainings) - - while remainings: - time.sleep(1) - current_size = len(remainings) - - if current_size != last_size or current_size == 0: - # Reset the unchanged duration if something has changed - unchanged_duration = 0 - last_size = current_size - else: - # Increment the duration if nothing has changed - unchanged_duration += 1 - - if unchanged_duration >= 240: + not_done = [True] + while len(not_done) > 0: + done, not_done = wait(futures, timeout=240, return_when=ALL_COMPLETED) + + if len(done) == 0: # Output warnings after 240 seconds of no change warn("No samples have finished testing in the last 240s") warn(f"{len(remainings)} samples to be tested: {remainings}") - unchanged_duration = 0 # Reset after warning threading.Thread(target=stucking_checker).start() From 6c01136cf7a54aae2b4489996e683a4e5a35607a Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Sat, 14 Sep 2024 20:09:45 +0200 Subject: [PATCH 145/325] Remove superfluous comment --- bigcodebench/evaluate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 0451d2d..187cfc2 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -209,7 +209,6 @@ def stucking_checker(): done, not_done = wait(futures, timeout=240, return_when=ALL_COMPLETED) if len(done) == 0: - # Output warnings after 240 seconds of no change warn("No samples have finished testing in the last 240s") warn(f"{len(remainings)} samples to be tested: {remainings}") From afc1f87a0685d0b21b834ac0d0769dc7c3147a7b Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Sat, 14 Sep 2024 23:45:55 +0200 Subject: [PATCH 146/325] Reset timer on progress --- bigcodebench/evaluate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 187cfc2..41c7cc0 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -6,7 +6,7 @@ import threading import time from collections import Counter, defaultdict -from concurrent.futures import ProcessPoolExecutor, as_completed, wait, ALL_COMPLETED +from concurrent.futures import ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED from datetime import datetime from typing import Any, Dict, List, Tuple from warnings import warn @@ -204,9 +204,9 @@ def evaluate(flags): assert len(completion_id) == len(problems), "Missing problems in samples" def stucking_checker(): - not_done = [True] + not_done = futures while len(not_done) > 0: - done, not_done = wait(futures, timeout=240, return_when=ALL_COMPLETED) + done, not_done = wait(not_done, timeout=240, return_when=FIRST_COMPLETED) if len(done) == 0: warn("No samples have finished testing in the last 240s") From 129e263459ed1775738064bda085b4d10d35931d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 16 Sep 2024 05:37:41 +0800 Subject: [PATCH 147/325] feat: add more models --- analysis/utils.py | 117 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index 41f50b0..4bac8d1 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1052,4 +1052,121 @@ "act_param": 72, "open-data": "None", }, + "gemini-1.5-pro-exp-0827": { + "name": "Gemini-1.5-Pro-Exp-0827", + "link": "https://deepmind.google/technologies/gemini/pro", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "gemini-1.5-flash-exp-0827": { + "name": "Gemini-1.5-Flash-Exp-0827", + "link": "https://deepmind.google/technologies/gemini/flash/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "microsoft/Phi-3.5-mini-instruct": { + "name": "Phi-3.5-Mini-Instruct", + "link": "https://huggingface.co/microsoft/Phi-3.5-mini-instruct", + "prompted": True, + "moe": False, + "size": 3.8, + "act_param": 3.8, + "open-data": "None", + }, + "abacusai/Dracarys-Llama-3.1-70B-Instruct": { + "name": "Dracarys-Llama-3.1-70B-Instruct", + "link": "https://huggingface.co/abacusai/Dracarys-Llama-3.1-70B-Instruct", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "abacusai/Dracarys-72B-Instruct": { + "name": "Dracarys-72B-Instruct", + "link": "https://huggingface.co/abacusai/Dracarys-72B-Instruct", + "prompted": True, + "moe": False, + "size": 72, + "act_param": 72, + "open-data": "None", + }, + "deepseek-coder-v2.5": { + "name": "DeepSeek-V2.5", + "link": "https://www.deepseek.com/", + "prompted": True, + "moe": True, + "size": 236, + "act_param": 21, + "open-data": "None", + }, + "CohereForAI/c4ai-command-r-08-2024": { + "name": "C4AI-Command-R-08-2024", + "link": "https://huggingface.co/CohereForAI/c4ai-command-r-08-2024", + "prompted": True, + "moe": False, + "size": 32.3, + "act_param": 32.3, + "open-data": "None", + }, + "CohereForAI/c4ai-command-r-plus-08-2024": { + "name": "C4AI-Command-R-Plus-08-2024", + "link": "https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024", + "prompted": True, + "moe": False, + "size": 104, + "act_param": 104, + "open-data": "None", + }, + "ayueei--yue-coder-9b-preview": { + "name": "Yi-Coder-9B-Chat", + "link": "https://huggingface.co/01-ai/Yi-Coder-9B-Chat", + "prompted": True, + "moe": False, + "size": 9, + "act_param": 9, + "open-data": "None", + }, + "mattshumer/ref_70_e3_prefill": { + "name": "Reflection-Llama-3.1-70B", + "link": "https://huggingface.co/mattshumer/ref_70_e3", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "mattshumer/ref_70_e3": { + "name": "Reflection-Llama-3.1-70B (Recommended Settings)", + "link": "https://huggingface.co/mattshumer/ref_70_e3", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "o1-preview-2024-09-12": { + "name": "o1-Preview-2024-09-12 (temperature=1)", + "link": "https://o1.ai/o1-preview", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "o1-mini-2024-09-12": { + "name": "o1-Mini-2024-09-12 (temperature=1)", + "link": "https://o1.ai/o1-preview", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, } \ No newline at end of file From 8f83db191bf0e0696ad81070d49e2c309f680423 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 2 Oct 2024 01:12:18 +0800 Subject: [PATCH 148/325] feat: refactor generate pipeline --- bigcodebench/generate.py | 115 +++++++++------- bigcodebench/model.py | 289 ++++++++++++++++++++++----------------- bigcodebench/sanitize.py | 9 +- 3 files changed, 236 insertions(+), 177 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 679300c..641c09d 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -4,6 +4,7 @@ from bigcodebench.model import DecoderBase, make_model from bigcodebench.data import get_bigcodebench, write_jsonl +from bigcodebench.sanitize import sanitize from rich.progress import ( BarColumn, MofNCompleteColumn, @@ -23,6 +24,7 @@ def codegen( n_samples=1, id_range=None, resume=True, + batch_size: int=-1, ): with Progress( TextColumn(f"BigCodeBench--{split.capitalize()} ({subset.capitalize()}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), @@ -41,65 +43,81 @@ def codegen( dirname = os.path.dirname(save_path) if not os.path.exists(dirname) and dirname != "": os.makedirs(dirname) + + batch_prompts = [] + batch_task_ids = [] + batch_nsamples = [] + batch_entry_points = [] + + # Read existing data once if resuming + existing_data = {} + if resume and os.path.exists(save_path): + with open(save_path, "r") as f: + for line in f: + item = json.loads(line) + existing_data[item["task_id"]] = existing_data.get(item["task_id"], 0) + 1 + for id_num, (task_id, task) in enumerate(p.track(dataset.items())): if id_range is not None: low, high = id_range - if id_num < low or id_num >= high: + if id_num < low: p.console.print(f"Skipping {task_id} as it is not in {id_range}") continue + if id_num > id_range[1]: + break p_name = task_id.replace("/", "_") - # read the existing file if save_path exists - if os.path.exists(save_path): - with open(save_path, "r") as f: - existing_data = f.read().splitlines() - log = f"Codegen: {p_name} @ {model}" - n_existing = 0 - if resume: - if os.path.exists(save_path): - n_existing = len([1 for line in existing_data if json.loads(line)["task_id"] == task_id]) - else: - n_existing = 0 + n_existing = existing_data.get(task_id, 0) + nsamples = n_samples - n_existing + + try: + prompt = task[f"{split}_prompt"] + except: + raise Exception(f"Invalid split {split} for bigcodebench-{subset}") + if strip_newlines: + prompt = prompt.strip("\n") + + if nsamples > 0: + batch_prompts.append(prompt) + batch_task_ids.append(task_id) + batch_nsamples.append(nsamples) + batch_entry_points.append(task["entry_point"]) + + log = f"Codegen: {p_name} @ {model}" if n_existing > 0: log += f" (resuming from {n_existing})" - - nsamples = n_samples - n_existing - p.console.print(log) - - sidx = n_samples - nsamples - while sidx < n_samples: - try: - prompt = task[f"{split}_prompt"] - except: - raise Exception(f"Invalid split {split}") - if strip_newlines: - prompt = prompt.strip("\n") + p.console.print(log) + + if (batch_size and len(batch_prompts) == batch_size) or id_num == len(dataset) - 1 or (id_range and id_num == id_range[1] - 1): + if not batch_prompts and id_num == len(dataset) - 1: + break outputs = model.codegen( - prompt, + batch_prompts, do_sample=not greedy, - num_samples=n_samples - sidx, + num_samples=max(batch_nsamples), ) assert outputs, "No outputs from model!" - if model.is_direct_completion(): - samples = [ - dict( - task_id=task_id, - solution=task["complete_prompt"]+completion - ) - for task_id, completion in zip([task_id]*len(outputs), outputs) - ] - else: - samples = [ - dict( - task_id=task_id, - solution=completion, - ) - for task_id, completion in zip([task_id]*len(outputs), outputs) - ] + + samples = [] + for task_id, content, entry_point, nsamples, task_outputs in zip(batch_task_ids, batch_prompts, batch_entry_points, batch_nsamples, outputs): + if model.is_direct_completion(): + samples.extend([ + dict(task_id=task_id, solution=sanitize(content+completion, entry_point)) + for completion in task_outputs[:nsamples] + ]) + else: + samples.extend([ + dict(task_id=task_id, solution=sanitize(completion, entry_point)) + for completion in task_outputs[:nsamples] + ]) print(f"Generated {len(samples)} samples") write_jsonl(save_path, samples, append=True) - sidx += len(outputs) + + # Clear batches + batch_prompts = [] + batch_task_ids = [] + batch_nsamples = [] def main(): @@ -113,6 +131,7 @@ def main(): parser.add_argument("--temperature", default=0.0, type=float) parser.add_argument("--greedy", action="store_true") parser.add_argument("--strip_newlines", action="store_true") + parser.add_argument("--direct_completion", action="store_true") parser.add_argument("--resume", action="store_true") parser.add_argument("--id_range", nargs=2, type=int) parser.add_argument("--backend", default="vllm", type=str, choices=["vllm", "hf", "openai", "mistral", "anthropic", "google"]) @@ -126,7 +145,6 @@ def main(): if args.greedy or (args.temperature == 0 and args.n_samples == 1): args.temperature = 0 - args.bs = 1 args.n_samples = 1 args.greedy = True print("Greedy decoding ON (--greedy): setting bs=1, n_samples=1, temperature=0") @@ -140,18 +158,20 @@ def main(): model_runner = make_model( model=args.model, backend=args.backend, - batch_size=args.bs, + subset=args.subset, + split=args.split, temperature=args.temperature, base_url=args.base_url, tp=args.tp, trust_remote_code=args.trust_remote_code, + direct_completion=args.direct_completion, tokenizer_name=args.tokenizer_name, tokenizer_legacy=args.tokenizer_legacy ) extra = "-" + args.subset if args.subset != "full" else "" if not args.save_path: - save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}.jsonl" + save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}-sanitized_calibrated.jsonl" else: save_path = args.save_path @@ -164,7 +184,8 @@ def main(): strip_newlines=args.strip_newlines, n_samples=args.n_samples, resume=args.resume, - id_range=args.id_range + id_range=args.id_range, + batch_size=args.bs ) diff --git a/bigcodebench/model.py b/bigcodebench/model.py index c5093b0..0327160 100644 --- a/bigcodebench/model.py +++ b/bigcodebench/model.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from typing import List from warnings import warn - +from tqdm import tqdm import openai try: @@ -55,9 +55,9 @@ def extra_eos_for_direct_completion(dataset) -> List[str]: _MAGIC_SPLITTER_ = "-[[]]-this-is-really-our-highest-priority-[[]]-" -def make_chat_prompt(prompt: str, tokenizer: AutoTokenizer) -> str: +def make_chat_prompt(prompt: str, subset: str, split: str, tokenizer: AutoTokenizer, direct_completion: bool = False) -> str: # directly return prompt if it does not have a tokenizer.chat_template - if tokenizer.chat_template is None: + if tokenizer.chat_template is None or direct_completion: return prompt prompt = f"""\ @@ -86,29 +86,33 @@ class DecoderBase(ABC): def __init__( self, name: str, - batch_size: int = 1, + subset: str, + split: str, temperature: float = 0.8, - max_new_tokens: int = 1280, + max_new_tokens: int = 5120, dtype: str = "bfloat16", # default + direct_completion: bool = False, trust_remote_code: bool = False, tokenizer_name: str = None, tokenizer_legacy: bool = False, ) -> None: print("Initializing a decoder model: {} ...".format(name)) self.name = name - self.batch_size = batch_size + self.subset = subset + self.split = split self.temperature = temperature self.eos = EOS self.skip_special_tokens = False self.max_new_tokens = max_new_tokens self.dtype = dtype + self.direct_completion = direct_completion self.trust_remote_code = trust_remote_code self.tokenizer_name = tokenizer_name self.tokenizer_legacy = tokenizer_legacy @abstractmethod def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: pass @@ -138,31 +142,32 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=self.tokenizer_legacy) if self.tokenizer.chat_template is None: self.eos += extra_eos_for_direct_completion(dataset) - self.llm = LLM(model=name, max_model_len=2048, **kwargs) + self.llm = LLM(model=name, max_model_len=self.max_new_tokens, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) def is_direct_completion(self) -> bool: - return self.tokenizer.chat_template is None + return self.tokenizer.chat_template is None or self.direct_completion def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: if do_sample: assert self.temperature > 0, "Temperature must be greater than 0!" - batch_size = min(self.batch_size, num_samples) vllm_outputs = self.llm.generate( - [prompt] * batch_size, + prompts, SamplingParams( + n=num_samples, temperature=self.temperature, max_tokens=self.max_new_tokens, top_p=0.95 if do_sample else 1.0, stop=self.eos, + skip_special_tokens=self.skip_special_tokens, ), - use_tqdm=False, + use_tqdm=True, ) - gen_strs = [x.outputs[0].text.replace("\t", " ") for x in vllm_outputs] + gen_strs = [[x.text.replace("\t", " ") for x in output.outputs] for output in vllm_outputs] return gen_strs @@ -173,10 +178,10 @@ def __init__(self, name: str, **kwargs) -> None: print(f"EOS strings: {self.eos}") def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: - prompt = make_chat_prompt(prompt, self.tokenizer) - return VllmDecoder.codegen(self, prompt, do_sample, num_samples) + prompts = [make_chat_prompt(prompt, self.subset, self.split, self.tokenizer, self.direct_completion) for prompt in prompts] + return VllmDecoder.codegen(self, prompts, do_sample, num_samples) class HfTorchDecoder(DecoderBase): @@ -204,17 +209,17 @@ def __init__(self, name: str, dataset: str, **kwargs): self.model = self.model.to(self.device) def is_direct_completion(self) -> bool: - return self.tokenizer.chat_template is None + return self.tokenizer.chat_template is None or self.direct_completion @torch.inference_mode() def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: if self.temperature == 0: assert not do_sample assert num_samples == 1 - input_tokens = self.tokenizer.encode(prompt, return_tensors="pt").to( + input_tokens = self.tokenizer.encode(prompts, return_tensors="pt").to( self.device ) kwargs = {} @@ -226,7 +231,7 @@ def codegen( input_tokens, max_new_tokens=self.max_new_tokens, do_sample=do_sample, - num_return_sequences=min(self.batch_size, num_samples), + num_return_sequences=num_samples, pad_token_id=self.tokenizer.eos_token_id, stop_strings=self.eos, tokenizer=self.tokenizer, @@ -257,10 +262,10 @@ def __init__(self, name: str, **kwargs): **kwargs, legacy=self.tokenizer_legacy) def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: - prompt = make_chat_prompt(prompt, self.tokenizer) - return HfTorchDecoder.codegen(self, prompt, do_sample, num_samples) + prompts = [make_chat_prompt(prompt, self.subset, self.split, self.tokenizer, self.direct_completion) for prompt in prompts] + return HfTorchDecoder.codegen(self, prompts, do_sample, num_samples) class OpenAIChatDecoder(DecoderBase): @@ -269,48 +274,50 @@ def __init__(self, name: str, base_url=None, **kwargs) -> None: self.client = openai.OpenAI(base_url=base_url) def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: if do_sample: assert self.temperature > 0, "Temperature must be positive for sampling" - batch_size = min(self.batch_size, num_samples) # construct prompt fmt = "json_object" if self.name == "gpt-4-1106-preview" else "text" - if fmt == "json_object": - message = r'Please complete the following code snippet by generating JSON like {"code": ""}' - else: - message = r"Please generate self-contained code to complete the following problem:" - - message += f"\n```python\n{prompt.strip()}\n```" - - ret = openai_request.make_auto_request( - self.client, - message=message, - model=self.name, - max_tokens=self.max_new_tokens, - temperature=self.temperature, - n=batch_size, - response_format={"type": fmt}, - ) - - outputs = [] - for item in ret.choices: - content = item.message.content - # if json serializable + all_outputs = [] + for prompt in tqdm(prompts): if fmt == "json_object": - try: - json_data = json.loads(content) - if json_data.get("code", None) is not None: - outputs.append(prompt + "\n" + json_data["code"]) - continue - - print(f"'code' field not found in: {json_data}") - except Exception as e: - print(e) - outputs.append(content) - - return outputs + message = r'Please provide a self-contained Python script by generating JSON like {"code": ""}' + else: + message = r"Please provide a self-contained Python script that solves the following problem in a markdown code block:" + + message += f"\n{prompt.strip()}\n" + + outputs = [] + while len(outputs) < num_samples: + ret = openai_request.make_auto_request( + self.client, + message=message, + model=self.name, + max_tokens=self.max_new_tokens, + temperature=self.temperature, + n=1, + response_format={"type": fmt}, + ) + for item in ret.choices: + content = item.message.content + # if json serializable + if fmt == "json_object": + try: + json_data = json.loads(content) + if json_data.get("code", None) is not None: + outputs.append(prompt + "\n" + json_data["code"]) + continue + + print(f"'code' field not found in: {json_data}") + except Exception as e: + print(e) + outputs.append(content) + all_outputs.append(outputs) + + return all_outputs def is_direct_completion(self) -> bool: return False @@ -332,26 +339,33 @@ def codegen( else: self.temperature = 0 - batch_size = min(self.batch_size, num_samples) + all_outputs = [] + + for prompt in prompts: + outputs = [] + message = f"""\ +Please provide a self-contained Python script that solves the following problem in a markdown code block: +{prompt.strip()} +""" - outputs = [] - for _ in range(batch_size): - ret = self.client.chat( - model=self.name, - messages=[ - ChatMessage( - role="user", - content="Please generate self-contained code to solve the following problem in a Python markdown block:" - + f"\n```python\n{prompt.strip()}\n```", - ) - ], - max_tokens=self.max_new_tokens, - **kwargs, - ) + for _ in range(num_samples): + ret = self.client.chat( + model=self.name, + messages=[ + ChatMessage( + role="user", + content=message, + ) + ], + max_tokens=self.max_new_tokens, + **kwargs, + ) - outputs.append(ret.choices[0].message.content) + outputs.append(ret.choices[0].message.content) - return outputs + all_outputs.append(outputs) + + return all_outputs def is_direct_completion(self) -> bool: return False @@ -378,28 +392,30 @@ def codegen( else: self.temperature = 0 - batch_size = min(self.batch_size, num_samples) - if not do_sample: - assert batch_size == 1, "Sampling only supports batch size of 1" - - outputs = [] - for _ in range(batch_size): - message = anthropic_request.make_auto_request( - client=self.client, - model=self.name, - messages=[ - { - "role": "user", - "content": "Please generate self-contained code to complete the following problem wrapped in a Python markdown block:" - + f"\n```python\n{prompt.strip()}\n```\n", - } - ], - max_tokens=self.max_new_tokens, - stop_sequences=["\n```\n", "\nif "], - **kwargs, - ) - outputs.append(message.content[0].text) - + all_outputs = [] + for prompt in tqdm(prompts): + outputs = [] + message = f"""\ +Please provide a self-contained Python script that solves the following problem in a markdown code block: +{prompt.strip()} +""" + for _ in range(num_samples): + ret = anthropic_request.make_auto_request( + client=self.client, + model=self.name, + messages=[ + { + "role": "user", + "content": message, + } + ], + max_tokens=self.max_new_tokens, + stop_sequences=["\n```\n", "\nif "], + **kwargs, + ) + outputs.append(ret.content[0].text) + + all_outputs.append(outputs) return outputs @@ -414,7 +430,7 @@ def is_direct_completion(self) -> bool: class GeminiDecoder(GoogleGenAIDecoder): def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: kwargs = {} if do_sample: @@ -424,10 +440,6 @@ def codegen( else: self.temperature = 0 - batch_size = min(self.batch_size, num_samples) - if not do_sample: - assert batch_size == 1, "Sampling only supports batch size of 1" - genai_config = genai.GenerationConfig( max_output_tokens=self.max_new_tokens, **kwargs, @@ -458,37 +470,48 @@ def codegen( model = genai.GenerativeModel(model_name=self.name, generation_config=genai_config, safety_settings=safety_settings) - outputs = [] - for _ in range(batch_size): - while True: - try: - response = model.generate_content( - "Please generate self-contained code to complete the following problem wrapped in a Python markdown block:" - + f"\n```python\n{prompt.strip()}\n```", - generation_config=genai_config + all_outputs = [] + + for prompt in tqdm(prompts): + outputs = [] + message = f"""\ +Please provide a self-contained Python script that solves the following problem in a markdown code block: +{prompt.strip()} +""" + + for _ in range(num_samples): + while True: + try: + response = model.generate_content( + message, + generation_config=genai_config ) - output = response.candidates[0].content.parts[0].text - outputs.append(output) - break - except Exception as e: - if "list index out of range" in str(e): - # append dummy response - outputs.append("NO_RESPONSE") + output = response.candidates[0].content.parts[0].text + outputs.append(output) break - else: - print(e) - continue + except Exception as e: + if "list index out of range" in str(e): + # append dummy response + outputs.append("NO_RESPONSE") + break + else: + print(e) + continue - return outputs + all_outputs.append(outputs) + + return all_outputs def make_model( model: str, backend: str, + subset: str, + split: str, dataset: str = "bigcodebench", - batch_size: int = 1, temperature: float = 0.0, tp=1, + direct_completion=False, base_url=None, trust_remote_code=False, tokenizer_name=None, @@ -497,10 +520,12 @@ def make_model( if backend == "vllm": return GeneralVllmDecoder( name=model, - batch_size=batch_size, + subset=subset, + split=split, temperature=temperature, dataset=dataset, tp=tp, + direct_completion=direct_completion, trust_remote_code=trust_remote_code, tokenizer_name=tokenizer_name, tokenizer_legacy=tokenizer_legacy, @@ -508,9 +533,11 @@ def make_model( elif backend == "hf": return GenenralHfTorchDecoder( name=model, - batch_size=batch_size, + subset=subset, + split=split, temperature=temperature, dataset=dataset, + direct_completion=direct_completion, trust_remote_code=trust_remote_code, tokenizer_name=tokenizer_name, tokenizer_legacy=tokenizer_legacy, @@ -518,25 +545,29 @@ def make_model( elif backend == "openai": return OpenAIChatDecoder( name=model, - batch_size=batch_size, + subset=subset, + split=split, temperature=temperature, base_url=base_url, ) elif backend == "mistral": return MistralChatDecoder( name=model, - batch_size=batch_size, + subset=subset, + split=split, temperature=temperature, ) elif backend == "anthropic": return AnthropicMessageDecoder( name=model, - batch_size=batch_size, + subset=subset, + split=split, temperature=temperature, ) elif backend == "google": return GeminiDecoder( name=model, - batch_size=batch_size, + subset=subset, + split=split, temperature=temperature, ) \ No newline at end of file diff --git a/bigcodebench/sanitize.py b/bigcodebench/sanitize.py index df9ed4e..5a8ab53 100644 --- a/bigcodebench/sanitize.py +++ b/bigcodebench/sanitize.py @@ -108,7 +108,7 @@ def has_return_statement(node: Node) -> bool: return False -def sanitize(code: str, entrypoint: Optional[str] = None) -> str: +def extract_target_code_or_empty(code: str, entrypoint: Optional[str] = None) -> str: code = code_extract(code.strip()) code_bytes = bytes(code, "utf8") parser = get_parser("python") @@ -179,6 +179,13 @@ def sanitize(code: str, entrypoint: Optional[str] = None) -> str: return sanitized_output +def sanitize(code: str, entrypoint: Optional[str] = None) -> str: + sanitized_code = extract_target_code_or_empty(code, entrypoint).strip() + if not sanitized_code: + return code_extract(code) + return sanitized_code + + def process_solution( sample_solution: Dict, dataset: Dict, From 8ffe6d3d69e6d38a7f4ca626e93064374746af3b Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 2 Oct 2024 01:20:01 +0800 Subject: [PATCH 149/325] add customized pass k --- bigcodebench/evaluate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 61e2a43..d39bd93 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -249,7 +249,7 @@ def stucking_checker(): pass_at_k = { f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() - for k in [1, 5, 10, 25, 100] + for k in flags.pass_k if total.min() >= k } @@ -327,6 +327,7 @@ def main(): ) parser.add_argument("--subset", default="hard", type=str, choices=["full", "hard"]) parser.add_argument("--samples", required=True, type=str) + parser.add_argument("--pass_k", nargs='+', type=int, default=[1, 5, 10], help="List of k values to use, e.g., --pass_k 1 5 10") parser.add_argument("--save_pass_rate", action="store_true") parser.add_argument("--parallel", default=None, type=int) parser.add_argument("--min-time-limit", default=1, type=float) From f4b5978f5254290021dee96c3cdb52514ed41ffd Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 4 Oct 2024 17:46:12 +0000 Subject: [PATCH 150/325] feat: refactor eval --- bigcodebench/data/bigcodebench.py | 2 +- bigcodebench/evaluate.py | 387 ++++++++++++++++-------------- bigcodebench/generate.py | 134 ++++++----- 3 files changed, 283 insertions(+), 240 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 979ad54..da2ad5d 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -14,7 +14,7 @@ BIGCODEBENCH_OVERRIDE_PATH = os.environ.get("BIGCODEBENCH_OVERRIDE_PATH", None) BIGCODEBENCH_HF = "bigcode/bigcodebench" -BIGCODEBENCH_VERSION = "v0.1.1" +BIGCODEBENCH_VERSION = "v0.1.2" def _ready_bigcodebench_path(subset="full", version="default") -> str: if BIGCODEBENCH_OVERRIDE_PATH: diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index d39bd93..a71e9dd 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -8,13 +8,15 @@ from collections import Counter, defaultdict from concurrent.futures import ProcessPoolExecutor, as_completed from datetime import datetime -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, Optional from warnings import warn +from gradio_client import Client, handle_file import numpy as np from termcolor import cprint from tqdm import tqdm +from bigcodebench.generate import run_codegen from bigcodebench.data import ( get_bigcodebench, get_bigcodebench_hash, @@ -109,156 +111,212 @@ def check_correctness( return ret -def evaluate(flags): - if flags.parallel is None: - n_workers = max(1, multiprocessing.cpu_count() // 2) - else: - n_workers = flags.parallel - - if flags.check_gt_only: - # bypass the samples - flags.samples = "__dummy__.jsonl" +def evaluate( + split: str, + subset: str, + samples: Optional[str] = None, + remote_execute: bool = True, + remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/", + pass_k: str = "1,5,10", + save_pass_rate: bool = True, + parallel: int = None, + min_time_limit: float = 1, + max_as_limit: int = 30*1024, + max_data_limit: int = 30*1024, + max_stack_limit: int = 10, + check_gt_only: bool = False, + no_gt: bool = False, + **model_kwargs, +): + if not samples and model_kwargs: + samples = run_codegen( + split=split, + subset=subset, + **model_kwargs, + ) + assert samples is not None, "No samples provided" - extra = flags.subset + "_" if flags.subset != "full" else "" - if os.path.isdir(flags.samples): - result_path = os.path.join(flags.samples, f"{extra}eval_results.json") - else: - assert flags.samples.endswith(".jsonl") - result_path = flags.samples.replace(".jsonl", f"_{extra}eval_results.json") - - problems = get_bigcodebench(subset=flags.subset) - dataset_hash = get_bigcodebench_hash(subset=flags.subset) + extra = subset + "_" if subset != "full" else "" - if not flags.no_gt: - expected_time = get_groundtruth(n_workers, problems, dataset_hash, flags.check_gt_only, flags.max_as_limit, flags.max_data_limit, flags.max_stack_limit, flags.min_time_limit) + if os.path.isdir(samples): + result_path = os.path.join(samples, f"{extra}eval_results.json") else: - expected_time = {task_id: None for task_id in problems} - - gt_pass_rate = np.mean([1 if v is not None else 0 for k, v in expected_time.items() if k in problems]) - failed_tasks = [k for k, v in expected_time.items() if v is None and k in problems] + assert samples.endswith(".jsonl") + result_path = samples.replace(".jsonl", f"_{extra}eval_results.json") - if os.path.isfile(result_path): - print(f"Load from previous results from {result_path}") - with open(result_path, "r") as f: - results = json.load(f) - - results = compatible_eval_result(results) + if remote_execute: + + client = Client(remote_execute_api) + results, pass_at_k = client.predict( + split=split, + subset=subset, + samples=handle_file(samples), + pass_k=pass_k, + parallel=parallel, + min_time_limit=min_time_limit, + max_as_limit=max_as_limit, + max_data_limit=max_data_limit, + max_stack_limit=max_stack_limit, + check_gt_only=check_gt_only, + no_gt=no_gt, + api_name="/predict" + ) + gt_pass_rate = pass_at_k["gt_pass_rate"] + failed_tasks = pass_at_k["failed_tasks"] + else: - if flags.check_gt_only: - if gt_pass_rate > 0.99: - cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") - else: - cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") + pass_k = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()] - if len(failed_tasks) > 0: - cprint(f"Failed tasks: {failed_tasks}", "red") - - return + if parallel is None: + n_workers = max(1, multiprocessing.cpu_count() // 2) + else: + n_workers = parallel + + if check_gt_only: + # bypass the samples + samples = "__dummy__.jsonl" + + problems = get_bigcodebench(subset=subset) + dataset_hash = get_bigcodebench_hash(subset=subset) - results = { - "date": datetime.now().strftime("%Y-%m-%d %H:%M"), - "eval": {}, - } - - with ProcessPoolExecutor(max_workers=n_workers) as executor: - futures = [] - completion_id = Counter() - n_samples = 0 - eval_results = defaultdict(list) # task_id -> - remainings = set() - - print("Reading samples...") - for sample in tqdm(load_solutions(flags.samples)): - task_id = sample["task_id"] + if not no_gt: + expected_time = get_groundtruth(n_workers, problems, dataset_hash, check_gt_only, max_as_limit, max_data_limit, max_stack_limit, min_time_limit) + else: + expected_time = {task_id: None for task_id in problems} + + gt_pass_rate = np.mean([1 if v is not None else 0 for k, v in expected_time.items() if k in problems]) + failed_tasks = [k for k, v in expected_time.items() if v is None and k in problems] + + if os.path.isfile(result_path): + print(f"Load from previous results from {result_path}") + with open(result_path, "r") as f: + results = json.load(f) + + results = compatible_eval_result(results) + else: + pass_at_k = dict() + + if check_gt_only: + + if gt_pass_rate > 0.99: + cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green") + else: + cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red") + + if len(failed_tasks) > 0: + cprint(f"Failed tasks: {failed_tasks}", "red") - if task_id not in problems: - warn( - f"Task {task_id} is found in the samples but not found in the dataset" - ) - continue - solution = ( - sample["solution"] - if "solution" in sample - else problems[task_id]["complete_prompt"] + sample["completion"] - ) - if "sanitized-calibrated" in flags.samples: - solution = problems[task_id]["code_prompt"] + "\n pass\n" + solution - remainings.add(sample["_identifier"]) - args = ( - completion_id[task_id], - problems[task_id], - solution, - flags.max_as_limit, - flags.max_data_limit, - flags.max_stack_limit, - sample["_identifier"], - flags.min_time_limit, - expected_time[task_id] if expected_time[task_id] else 20 - ) - futures.append(executor.submit(check_correctness, *args)) - completion_id[task_id] += 1 - n_samples += 1 - - assert n_samples == len(remainings), "Missing problems in unfinished" - assert len(completion_id) == len(problems), "Missing problems in samples" - - def stucking_checker(): - while remainings: - last_size = len(remainings) - time.sleep(240) - if last_size != len(remainings) or len(remainings) == 0: + else: + results = { + "date": datetime.now().strftime("%Y-%m-%d %H:%M"), + "eval": {}, + } + + with ProcessPoolExecutor(max_workers=n_workers) as executor: + futures = [] + completion_id = Counter() + n_samples = 0 + eval_results = defaultdict(list) # task_id -> + remainings = set() + + print("Reading samples...") + for sample in tqdm(load_solutions(samples)): + task_id = sample["task_id"] + + if task_id not in problems: + warn( + f"Task {task_id} is found in the samples but not found in the dataset" + ) + continue + solution = ( + sample["solution"] + if "solution" in sample + else problems[task_id]["complete_prompt"] + sample["completion"] + ) + if "sanitized-calibrated" in samples: + solution = problems[task_id]["code_prompt"] + "\n pass\n" + solution + remainings.add(sample["_identifier"]) + args = ( + completion_id[task_id], + problems[task_id], + solution, + max_as_limit, + max_data_limit, + max_stack_limit, + sample["_identifier"], + min_time_limit, + expected_time[task_id] if expected_time[task_id] else 20 + ) + futures.append(executor.submit(check_correctness, *args)) + completion_id[task_id] += 1 + n_samples += 1 + + assert n_samples == len(remainings), "Missing problems in unfinished" + assert len(completion_id) == len(problems), "Missing problems in samples" + + def stucking_checker(): + while remainings: + last_size = len(remainings) + time.sleep(240) + if last_size != len(remainings) or len(remainings) == 0: + continue + # Potential stucking + warn("No samples had finished testing in the last 240s") + warn(f"{len(remainings)} samples to be tested: {remainings}") + + threading.Thread(target=stucking_checker).start() + + for future in tqdm(as_completed(futures), total=n_samples): + result = future.result() + remainings.remove(result["_identifier"]) + eval_results[result["task_id"]].append(result) + + # sort the results for each problem by completion_id + for task_id, task_results in eval_results.items(): + task_results.sort(key=lambda x: x["completion_id"]) + results["eval"][task_id] = [] + for res in task_results: + stat, details = res["base"] + results["eval"][task_id].append( + { + "task_id": task_id, + "solution": res["solution"], + "status": stat, + "details": details, + } + ) + + # Calculate pass@k. + total = np.array([len(r) for k, r in results["eval"].items() if k in problems]) + base_correct = [] + + for key, res in results["eval"].items(): + if key not in problems: continue - # Potential stucking - warn("No samples had finished testing in the last 240s") - warn(f"{len(remainings)} samples to be tested: {remainings}") - - threading.Thread(target=stucking_checker).start() - - for future in tqdm(as_completed(futures), total=n_samples): - result = future.result() - remainings.remove(result["_identifier"]) - eval_results[result["task_id"]].append(result) - - # sort the results for each problem by completion_id - for task_id, task_results in eval_results.items(): - task_results.sort(key=lambda x: x["completion_id"]) - results["eval"][task_id] = [] - for res in task_results: - stat, details = res["base"] - results["eval"][task_id].append( - { - "task_id": task_id, - "solution": res["solution"], - "status": stat, - "details": details, - } - ) - - # Calculate pass@k. - total = np.array([len(r) for k, r in results["eval"].items() if k in problems]) - base_correct = [] - - for key, res in results["eval"].items(): - if key not in problems: - continue - bc = sum([r["status"] == PASS for r in res]) - base_correct.append(bc) - - base_correct = np.array(base_correct) - - pass_at_k = { - f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() - for k in flags.pass_k - if total.min() >= k - } - - mode = "-calibrated" if "sanitized-calibrated" in flags.samples else "" - extra = flags.subset.capitalize() - flags.split = flags.split.capitalize() - cprint(f"BigCodeBench-{flags.split}{mode} ({extra})", "green") + bc = sum([r["status"] == PASS for r in res]) + base_correct.append(bc) + + base_correct = np.array(base_correct) + + pass_at_k.update({ + f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() + for k in pass_k + if total.min() >= k + }) + + pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0] + pass_at_k["split"] = split + pass_at_k["subset"] = subset + pass_at_k["calibrated"] = "sanitized-calibrated" in samples + pass_at_k["gt_pass_rate"] = gt_pass_rate + pass_at_k["failed_tasks"] = failed_tasks + + extra = subset.capitalize() + split = split.capitalize() + cprint(f"BigCodeBench-{split} ({extra})", "green") - if flags.no_gt: + if no_gt: cprint(f"Groundtruth is not checked", "yellow") else: if gt_pass_rate > 0.99: @@ -270,7 +328,8 @@ def stucking_checker(): cprint(f"Failed tasks: {failed_tasks}", "red") for k, v in pass_at_k.items(): - cprint(f"{k}:\t{v:.3f}", "green") + if k.startswith("pass@"): + cprint(f"{k}:\t{v:.3f}", "green") # save results if os.path.isfile(result_path): @@ -291,15 +350,8 @@ def stucking_checker(): with open(result_path, "w") as f: json.dump(results, f, indent=2) - if flags.save_pass_rate: + if save_pass_rate: pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") - pass_at_k["model"] = os.path.basename(flags.samples).split("--bigcodebench-")[0] - pass_at_k["calibrated"] = "sanitized-calibrated" in flags.samples - pass_at_k["subset"] = flags.subset - - def save_pass_at_k(): - with open(pass_at_k_path, "w") as f: - json.dump(pass_at_k, f, indent=2) if os.path.isfile(pass_at_k_path): saved_pass_at_k = json.load(open(pass_at_k_path, "r")) @@ -314,36 +366,21 @@ def save_pass_at_k(): print(f"Save pass@k to {pass_at_k_path}? [Y/N]") decision = input() if decision.lower() == "y": - save_pass_at_k() - - else: - save_pass_at_k() + new_path = result_path + ".bak" + while os.path.isfile(new_path): + new_path += ".bak" + os.rename(pass_at_k_path, new_path) + print(f"Backup {pass_at_k_path} to {new_path}") + + if not os.path.isfile(pass_at_k_path): + with open(pass_at_k_path, "w") as f: + json.dump(pass_at_k, f, indent=2) def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--split", required=True, type=str, choices=["complete", "instruct"] - ) - parser.add_argument("--subset", default="hard", type=str, choices=["full", "hard"]) - parser.add_argument("--samples", required=True, type=str) - parser.add_argument("--pass_k", nargs='+', type=int, default=[1, 5, 10], help="List of k values to use, e.g., --pass_k 1 5 10") - parser.add_argument("--save_pass_rate", action="store_true") - parser.add_argument("--parallel", default=None, type=int) - parser.add_argument("--min-time-limit", default=1, type=float) - parser.add_argument("--max-as-limit", default=30*1024, type=int) - parser.add_argument("--max-data-limit", default=30*1024, type=int) - parser.add_argument("--max-stack-limit", default=10, type=int) - parser.add_argument( - "--check-gt-only", action="store_true", help="Check the ground truth" - ) - parser.add_argument( - "--no-gt", action="store_true", help="Skip the ground truth" - ) - args = parser.parse_args() - - evaluate(args) + from fire import Fire + Fire(evaluate) if __name__ == "__main__": main() diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 641c09d..dd40cef 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -1,6 +1,7 @@ import os import json import argparse +from typing import Optional, Tuple from bigcodebench.model import DecoderBase, make_model from bigcodebench.data import get_bigcodebench, write_jsonl @@ -16,7 +17,7 @@ def codegen( model: DecoderBase, - save_path: str, + target_path: str, split: str, subset="full", greedy=False, @@ -39,8 +40,8 @@ def codegen( if model.is_direct_completion() and split == "instruct": raise Exception("Base model does not support direct completion for instruct tasks") - # create save_path if it doesn't exist, e.g., a/b.jsonl - dirname = os.path.dirname(save_path) + # create target_path if it doesn't exist, e.g., a/b.jsonl + dirname = os.path.dirname(target_path) if not os.path.exists(dirname) and dirname != "": os.makedirs(dirname) @@ -51,8 +52,8 @@ def codegen( # Read existing data once if resuming existing_data = {} - if resume and os.path.exists(save_path): - with open(save_path, "r") as f: + if resume and os.path.exists(target_path): + with open(target_path, "r") as f: for line in f: item = json.loads(line) existing_data[item["task_id"]] = existing_data.get(item["task_id"], 0) + 1 @@ -103,16 +104,17 @@ def codegen( for task_id, content, entry_point, nsamples, task_outputs in zip(batch_task_ids, batch_prompts, batch_entry_points, batch_nsamples, outputs): if model.is_direct_completion(): samples.extend([ - dict(task_id=task_id, solution=sanitize(content+completion, entry_point)) + dict(task_id=task_id, solution=sanitize(content+completion, entry_point), raw_solution=content+completion) for completion in task_outputs[:nsamples] ]) else: samples.extend([ - dict(task_id=task_id, solution=sanitize(completion, entry_point)) + dict(task_id=task_id, solution=sanitize(completion, entry_point), raw_solution=completion) for completion in task_outputs[:nsamples] ]) + print(f"Generated {len(samples)} samples") - write_jsonl(save_path, samples, append=True) + write_jsonl(target_path, samples, append=True) # Clear batches batch_prompts = [] @@ -120,74 +122,78 @@ def codegen( batch_nsamples = [] -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--model", required=True, type=str) - parser.add_argument("--split", required=True, type=str, choices=["complete", "instruct"]) - parser.add_argument("--subset", default="full", type=str, choices=["full", "hard"]) - parser.add_argument("--save_path", default=None, type=str) - parser.add_argument("--bs", default=1, type=int) - parser.add_argument("--n_samples", default=1, type=int) - parser.add_argument("--temperature", default=0.0, type=float) - parser.add_argument("--greedy", action="store_true") - parser.add_argument("--strip_newlines", action="store_true") - parser.add_argument("--direct_completion", action="store_true") - parser.add_argument("--resume", action="store_true") - parser.add_argument("--id_range", nargs=2, type=int) - parser.add_argument("--backend", default="vllm", type=str, choices=["vllm", "hf", "openai", "mistral", "anthropic", "google"]) - parser.add_argument("--base_url", default=None, type=str) - parser.add_argument("--tp", default=1, type=int) - parser.add_argument("--trust_remote_code", action="store_true") - parser.add_argument("--tokenizer_legacy", action="store_true") - parser.add_argument("--tokenizer_name", default=None, type=str) - - args = parser.parse_args() - - if args.greedy or (args.temperature == 0 and args.n_samples == 1): - args.temperature = 0 - args.n_samples = 1 - args.greedy = True +def run_codegen( + model: str, + split: str, + subset: str, + root: str = "bcb_results", + bs: Optional[int] = None, + n_samples: int = 1, + temperature: float = 0.0, + greedy: bool = False, + strip_newlines: bool = False, + direct_completion: bool = False, + resume: bool = True, + id_range: Tuple[int, int] = None, + backend: str = "vllm", + base_url: str = None, + tp: int = 1, + trust_remote_code: bool = False, + tokenizer_name: str = None, + tokenizer_legacy: bool = False, +): + + if greedy or (temperature == 0 and n_samples == 1): + temperature = 0 + n_samples = 1 + greedy = True print("Greedy decoding ON (--greedy): setting bs=1, n_samples=1, temperature=0") - if args.id_range is not None: - assert len(args.id_range) == 2, "id_range must be a list of length 2" - assert args.id_range[0] < args.id_range[1], "id_range must be increasing" - args.id_range = tuple(args.id_range) + if id_range is not None: + assert len(id_range) == 2, "id_range must be a list of length 2" + assert id_range[0] < id_range[1], "id_range must be increasing" + id_range = tuple(id_range) # Make dir for codes generated by each model model_runner = make_model( - model=args.model, - backend=args.backend, - subset=args.subset, - split=args.split, - temperature=args.temperature, - base_url=args.base_url, - tp=args.tp, - trust_remote_code=args.trust_remote_code, - direct_completion=args.direct_completion, - tokenizer_name=args.tokenizer_name, - tokenizer_legacy=args.tokenizer_legacy + model=model, + backend=backend, + subset=subset, + split=split, + temperature=temperature, + base_url=base_url, + tp=tp, + trust_remote_code=trust_remote_code, + direct_completion=direct_completion, + tokenizer_name=tokenizer_name, + tokenizer_legacy=tokenizer_legacy ) - extra = "-" + args.subset if args.subset != "full" else "" - if not args.save_path: - save_path = args.model.replace("/", "--") + f"--bigcodebench{extra}-{args.split}--{args.backend}-{args.temperature}-{args.n_samples}-sanitized_calibrated.jsonl" - else: - save_path = args.save_path + extra = "-" + subset if subset != "full" else "" + identifier = model.replace("/", "--") + f"--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" + + target_path = os.path.join(root, identifier) codegen( model=model_runner, - save_path=save_path, - split=args.split, - subset=args.subset, - greedy=args.greedy, - strip_newlines=args.strip_newlines, - n_samples=args.n_samples, - resume=args.resume, - id_range=args.id_range, - batch_size=args.bs + target_path=target_path, + split=split, + subset=subset, + greedy=greedy, + strip_newlines=strip_newlines, + n_samples=n_samples, + resume=resume, + id_range=id_range, + batch_size=bs ) + return target_path + + +def main(): + from fire import Fire + Fire(run_codegen) + if __name__ == "__main__": main() From 717835f76a98dd5ca00577df64e31c1fe93577a5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 06:00:53 +0800 Subject: [PATCH 151/325] feat: refactor: model provider --- bigcodebench/evaluate.py | 14 +- bigcodebench/gen/util/anthropic_request.py | 2 +- bigcodebench/gen/util/google_request.py | 45 ++ bigcodebench/gen/util/mistral_request.py | 12 + bigcodebench/generate.py | 8 + bigcodebench/model.py | 573 --------------------- bigcodebench/provider/__init__.py | 100 ++++ bigcodebench/provider/anthropic.py | 54 ++ bigcodebench/provider/base.py | 49 ++ bigcodebench/provider/google.py | 58 +++ bigcodebench/provider/hf.py | 95 ++++ bigcodebench/provider/mistral.py | 50 ++ bigcodebench/provider/openai.py | 46 ++ bigcodebench/provider/utility.py | 65 +++ bigcodebench/provider/vllm.py | 67 +++ 15 files changed, 656 insertions(+), 582 deletions(-) create mode 100644 bigcodebench/gen/util/google_request.py create mode 100644 bigcodebench/gen/util/mistral_request.py delete mode 100644 bigcodebench/model.py create mode 100644 bigcodebench/provider/__init__.py create mode 100644 bigcodebench/provider/anthropic.py create mode 100644 bigcodebench/provider/base.py create mode 100644 bigcodebench/provider/google.py create mode 100644 bigcodebench/provider/hf.py create mode 100644 bigcodebench/provider/mistral.py create mode 100644 bigcodebench/provider/openai.py create mode 100644 bigcodebench/provider/utility.py create mode 100644 bigcodebench/provider/vllm.py diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index f9d9dbf..1cee1b4 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -115,7 +115,7 @@ def evaluate( split: str, subset: str, samples: Optional[str] = None, - remote_execute: bool = True, + local_execute: bool = False, remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/", pass_k: str = "1,5,10", save_pass_rate: bool = True, @@ -135,16 +135,14 @@ def evaluate( **model_kwargs, ) assert samples is not None, "No samples provided" - - extra = subset + "_" if subset != "full" else "" - + if os.path.isdir(samples): - result_path = os.path.join(samples, f"{extra}eval_results.json") + result_path = os.path.join(samples, "eval_results.json") else: assert samples.endswith(".jsonl") - result_path = samples.replace(".jsonl", f"_{extra}eval_results.json") + result_path = samples.replace(".jsonl", "_eval_results.json") - if remote_execute: + if not local_execute: client = Client(remote_execute_api) results, pass_at_k = client.predict( @@ -351,7 +349,7 @@ def stucking_checker(): json.dump(results, f, indent=2) if save_pass_rate: - pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json") + pass_at_k_path = result_path.replace("eval_results.json", "pass_at_k.json") if os.path.isfile(pass_at_k_path): saved_pass_at_k = json.load(open(pass_at_k_path, "r")) diff --git a/bigcodebench/gen/util/anthropic_request.py b/bigcodebench/gen/util/anthropic_request.py index 06c86d5..e53feab 100644 --- a/bigcodebench/gen/util/anthropic_request.py +++ b/bigcodebench/gen/util/anthropic_request.py @@ -44,4 +44,4 @@ def make_auto_request(client: anthropic.Client, *args, **kwargs) -> Message: print(e) signal.alarm(0) time.sleep(1) - return ret + return ret \ No newline at end of file diff --git a/bigcodebench/gen/util/google_request.py b/bigcodebench/gen/util/google_request.py new file mode 100644 index 0000000..8a88842 --- /dev/null +++ b/bigcodebench/gen/util/google_request.py @@ -0,0 +1,45 @@ +import time + +import google.generativeai as genai +from google.api_core.exceptions import GoogleAPICallError, ResourceExhausted + + +def make_request( + client: genai.GenerativeModel, temperature, messages, max_new_tokens=2048 +) -> genai.types.GenerateContentResponse: + messages = [{"role": m["role"], "parts": [m["content"]]} for m in messages] + response = client.generate_content( + messages, + generation_config=genai.types.GenerationConfig( + candidate_count=1, + max_output_tokens=max_new_tokens, + temperature=temperature, + ), + safety_settings=[ + {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"}, + {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"}, + {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"}, + {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}, + ], + ) + + return response.text + + +def make_auto_request(*args, **kwargs) -> genai.types.GenerateContentResponse: + ret = None + while ret is None: + try: + ret = make_request(*args, **kwargs) + except ResourceExhausted as e: + print("Rate limit exceeded. Waiting...", e.message) + time.sleep(10) + except GoogleAPICallError as e: + print(e.message) + time.sleep(1) + except Exception as e: + print("Unknown error. Waiting...") + print(e) + time.sleep(1) + return ret + diff --git a/bigcodebench/gen/util/mistral_request.py b/bigcodebench/gen/util/mistral_request.py new file mode 100644 index 0000000..e61fce7 --- /dev/null +++ b/bigcodebench/gen/util/mistral_request.py @@ -0,0 +1,12 @@ +from mistralai.client import MistralClient +from mistralai.models.chat_completion import ChatMessage + +def make_auto_request(client: MistralClient, *args, **kwargs) -> ChatMessage: + ret = None + while ret is None: + try: + ret = client.chat(*args, **kwargs) + except Exception as e: + print(e) + time.sleep(1) + return ret \ No newline at end of file diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index dd40cef..d1f22d2 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -154,6 +154,12 @@ def run_codegen( assert id_range[0] < id_range[1], "id_range must be increasing" id_range = tuple(id_range) + # Make project dir + os.makedirs(root, exist_ok=True) + + instruction_prefix = "Please provide a self-contained Python script that solves the following problem in a markdown code block:" + response_prefix = "Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:" + # Make dir for codes generated by each model model_runner = make_model( model=model, @@ -161,6 +167,8 @@ def run_codegen( subset=subset, split=split, temperature=temperature, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, base_url=base_url, tp=tp, trust_remote_code=trust_remote_code, diff --git a/bigcodebench/model.py b/bigcodebench/model.py deleted file mode 100644 index 0327160..0000000 --- a/bigcodebench/model.py +++ /dev/null @@ -1,573 +0,0 @@ -import json -import os -from abc import ABC, abstractmethod -from typing import List -from warnings import warn -from tqdm import tqdm -import openai - -try: - import anthropic - - from bigcodebench.gen.util import anthropic_request -except ImportError: - warn("Anthropic decoder will not work. Fix by `pip install anthropic`") - -# mistral.ai -try: - from mistralai.client import MistralClient - from mistralai.models.chat_completion import ChatMessage -except ImportError: - warn("MistralAI decoder will not work. Fix by `pip install mistralai`") - -try: - import google.generativeai as genai -except ImportError: - warn("GoogleGenAI decoder will not work. Fix by `pip install google-generativeai`") - -import torch -from transformers import AutoModelForCausalLM, AutoTokenizer - -try: - from vllm import LLM, SamplingParams -except ImportError: - warn("VLLM decoder will not work. Fix by `pip install vllm`") - -from bigcodebench.gen.util import openai_request - -EOS = [ - "<|endoftext|>", - "<|endofmask|>", - "", - "\nif __name__", - "\ndef main(", - "\nprint(", -] - - -def extra_eos_for_direct_completion(dataset) -> List[str]: - if dataset.lower() == "bigcodebench": - return ["\ndef ", "\nclass ", "\nimport ", "\nfrom ", "\nassert "] - raise ValueError(f"Unknown dataset: {dataset}") - - -# some random words which serves as the splitter -_MAGIC_SPLITTER_ = "-[[]]-this-is-really-our-highest-priority-[[]]-" - - -def make_chat_prompt(prompt: str, subset: str, split: str, tokenizer: AutoTokenizer, direct_completion: bool = False) -> str: - # directly return prompt if it does not have a tokenizer.chat_template - if tokenizer.chat_template is None or direct_completion: - return prompt - - prompt = f"""\ -Please provide a self-contained Python script that solves the following problem in a markdown code block: -``` -{prompt.strip()} -``` -""" - response = f"""\ -Below is a Python script with a self-contained function that solves the problem and passes corresponding tests: -```python -{_MAGIC_SPLITTER_} -``` -""" - prompt = tokenizer.apply_chat_template( - [ - {"role": "user", "content": prompt}, - {"role": "assistant", "content": response}, - ], - tokenize=False, - ).split(_MAGIC_SPLITTER_)[0] - return prompt - - -class DecoderBase(ABC): - def __init__( - self, - name: str, - subset: str, - split: str, - temperature: float = 0.8, - max_new_tokens: int = 5120, - dtype: str = "bfloat16", # default - direct_completion: bool = False, - trust_remote_code: bool = False, - tokenizer_name: str = None, - tokenizer_legacy: bool = False, - ) -> None: - print("Initializing a decoder model: {} ...".format(name)) - self.name = name - self.subset = subset - self.split = split - self.temperature = temperature - self.eos = EOS - self.skip_special_tokens = False - self.max_new_tokens = max_new_tokens - self.dtype = dtype - self.direct_completion = direct_completion - self.trust_remote_code = trust_remote_code - self.tokenizer_name = tokenizer_name - self.tokenizer_legacy = tokenizer_legacy - - @abstractmethod - def codegen( - self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - pass - - @abstractmethod - def is_direct_completion(self) -> bool: - pass - - def __repr__(self) -> str: - return self.name - - def __str__(self) -> str: - return self.name - - -class VllmDecoder(DecoderBase): - def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: - super().__init__(name, **kwargs) - - kwargs = { - "tensor_parallel_size": int(os.getenv("VLLM_N_GPUS", tp)), - "dtype": self.dtype, - "trust_remote_code": self.trust_remote_code, - } - if self.tokenizer_name is None: - self.tokenizer_name = self.name - - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=self.tokenizer_legacy) - if self.tokenizer.chat_template is None: - self.eos += extra_eos_for_direct_completion(dataset) - self.llm = LLM(model=name, max_model_len=self.max_new_tokens, **kwargs) - self.llm.set_tokenizer(tokenizer=self.tokenizer) - - def is_direct_completion(self) -> bool: - return self.tokenizer.chat_template is None or self.direct_completion - - def codegen( - self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - if do_sample: - assert self.temperature > 0, "Temperature must be greater than 0!" - - vllm_outputs = self.llm.generate( - prompts, - SamplingParams( - n=num_samples, - temperature=self.temperature, - max_tokens=self.max_new_tokens, - top_p=0.95 if do_sample else 1.0, - stop=self.eos, - skip_special_tokens=self.skip_special_tokens, - ), - use_tqdm=True, - ) - - gen_strs = [[x.text.replace("\t", " ") for x in output.outputs] for output in vllm_outputs] - return gen_strs - - -class GeneralVllmDecoder(VllmDecoder): - def __init__(self, name: str, **kwargs) -> None: - super().__init__(name, **kwargs) - self.eos += ["\n```\n"] - print(f"EOS strings: {self.eos}") - - def codegen( - self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - prompts = [make_chat_prompt(prompt, self.subset, self.split, self.tokenizer, self.direct_completion) for prompt in prompts] - return VllmDecoder.codegen(self, prompts, do_sample, num_samples) - - -class HfTorchDecoder(DecoderBase): - def __init__(self, name: str, dataset: str, **kwargs): - super().__init__(name=name, **kwargs) - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - kwargs = {} - kwargs["device_map"] = "auto" - kwargs["trust_remote_code"] = self.trust_remote_code - # string to torch dtype - kwargs["torch_dtype"] = getattr(torch, self.dtype) - self.skip_special_tokens = True - - print(f"{kwargs = }", self.tokenizer_name) - if self.tokenizer_name is None: - self.tokenizer_name = self.name - - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=self.tokenizer_legacy) - - if self.tokenizer.chat_template is None: - self.eos += extra_eos_for_direct_completion(dataset) - - self.model = AutoModelForCausalLM.from_pretrained(name, **kwargs) - self.model = self.model.to(self.device) - - def is_direct_completion(self) -> bool: - return self.tokenizer.chat_template is None or self.direct_completion - - @torch.inference_mode() - def codegen( - self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - if self.temperature == 0: - assert not do_sample - assert num_samples == 1 - - input_tokens = self.tokenizer.encode(prompts, return_tensors="pt").to( - self.device - ) - kwargs = {} - if do_sample: - kwargs["top_p"] = 0.95 - kwargs["temperature"] = self.temperature - - outputs = self.model.generate( - input_tokens, - max_new_tokens=self.max_new_tokens, - do_sample=do_sample, - num_return_sequences=num_samples, - pad_token_id=self.tokenizer.eos_token_id, - stop_strings=self.eos, - tokenizer=self.tokenizer, - **kwargs, - ) - - gen_strs = self.tokenizer.batch_decode( - outputs[:, input_tokens.size(-1) :], - skip_special_tokens=self.skip_special_tokens, - ) - outputs = [] - # removes eos tokens. - for output in gen_strs: - min_index = 10000 - for eos in self.eos: - if eos in output: - min_index = min(min_index, output.index(eos)) - outputs.append(output[:min_index].replace("\t", " ")) - return outputs - - -class GenenralHfTorchDecoder(HfTorchDecoder): - def __init__(self, name: str, **kwargs): - super().__init__(name=name, **kwargs) - self.eos += ["\n```\n"] - print(f"EOS strings: {self.eos}") - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name, - **kwargs, legacy=self.tokenizer_legacy) - - def codegen( - self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - prompts = [make_chat_prompt(prompt, self.subset, self.split, self.tokenizer, self.direct_completion) for prompt in prompts] - return HfTorchDecoder.codegen(self, prompts, do_sample, num_samples) - - -class OpenAIChatDecoder(DecoderBase): - def __init__(self, name: str, base_url=None, **kwargs) -> None: - super().__init__(name, **kwargs) - self.client = openai.OpenAI(base_url=base_url) - - def codegen( - self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - if do_sample: - assert self.temperature > 0, "Temperature must be positive for sampling" - - # construct prompt - fmt = "json_object" if self.name == "gpt-4-1106-preview" else "text" - all_outputs = [] - for prompt in tqdm(prompts): - if fmt == "json_object": - message = r'Please provide a self-contained Python script by generating JSON like {"code": ""}' - else: - message = r"Please provide a self-contained Python script that solves the following problem in a markdown code block:" - - message += f"\n{prompt.strip()}\n" - - outputs = [] - while len(outputs) < num_samples: - ret = openai_request.make_auto_request( - self.client, - message=message, - model=self.name, - max_tokens=self.max_new_tokens, - temperature=self.temperature, - n=1, - response_format={"type": fmt}, - ) - for item in ret.choices: - content = item.message.content - # if json serializable - if fmt == "json_object": - try: - json_data = json.loads(content) - if json_data.get("code", None) is not None: - outputs.append(prompt + "\n" + json_data["code"]) - continue - - print(f"'code' field not found in: {json_data}") - except Exception as e: - print(e) - outputs.append(content) - all_outputs.append(outputs) - - return all_outputs - - def is_direct_completion(self) -> bool: - return False - - -class MistralChatDecoder(DecoderBase): - def __init__(self, name: str, **kwargs) -> None: - super().__init__(name, **kwargs) - self.client = MistralClient(api_key=os.getenv("MISTRAL_API_KEY")) - - def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - kwargs = {} - if do_sample: - assert self.temperature > 0, "Temperature must be positive for sampling" - kwargs["top_p"] = 0.95 - kwargs["temperature"] = self.temperature - else: - self.temperature = 0 - - all_outputs = [] - - for prompt in prompts: - outputs = [] - message = f"""\ -Please provide a self-contained Python script that solves the following problem in a markdown code block: -{prompt.strip()} -""" - - for _ in range(num_samples): - ret = self.client.chat( - model=self.name, - messages=[ - ChatMessage( - role="user", - content=message, - ) - ], - max_tokens=self.max_new_tokens, - **kwargs, - ) - - outputs.append(ret.choices[0].message.content) - - all_outputs.append(outputs) - - return all_outputs - - def is_direct_completion(self) -> bool: - return False - - -class AnthropicDecoder(DecoderBase, ABC): - def __init__(self, name: str, **kwargs) -> None: - super().__init__(name, **kwargs) - self.client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_KEY")) - - def is_direct_completion(self) -> bool: - return False - - -class AnthropicMessageDecoder(AnthropicDecoder): - def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - kwargs = {} - if do_sample: - assert self.temperature > 0, "Temperature must be positive for sampling" - kwargs["top_p"] = 0.95 - kwargs["temperature"] = self.temperature - else: - self.temperature = 0 - - all_outputs = [] - for prompt in tqdm(prompts): - outputs = [] - message = f"""\ -Please provide a self-contained Python script that solves the following problem in a markdown code block: -{prompt.strip()} -""" - for _ in range(num_samples): - ret = anthropic_request.make_auto_request( - client=self.client, - model=self.name, - messages=[ - { - "role": "user", - "content": message, - } - ], - max_tokens=self.max_new_tokens, - stop_sequences=["\n```\n", "\nif "], - **kwargs, - ) - outputs.append(ret.content[0].text) - - all_outputs.append(outputs) - return outputs - - -class GoogleGenAIDecoder(DecoderBase, ABC): - def __init__(self, name: str, **kwargs) -> None: - super().__init__(name, **kwargs) - genai.configure(api_key=os.environ['GOOGLE_API_KEY']) - - def is_direct_completion(self) -> bool: - return False - - -class GeminiDecoder(GoogleGenAIDecoder): - def codegen( - self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - ) -> List[str]: - kwargs = {} - if do_sample: - assert self.temperature > 0, "Temperature must be positive for sampling" - kwargs["top_p"] = 0.95 - kwargs["temperature"] = self.temperature - else: - self.temperature = 0 - - genai_config = genai.GenerationConfig( - max_output_tokens=self.max_new_tokens, - **kwargs, - ) - - safety_settings = [ - { - "category": "HARM_CATEGORY_DANGEROUS", - "threshold": "BLOCK_NONE", - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_NONE", - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_NONE", - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_NONE", - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_NONE", - }, - ] - - model = genai.GenerativeModel(model_name=self.name, generation_config=genai_config, safety_settings=safety_settings) - - all_outputs = [] - - for prompt in tqdm(prompts): - outputs = [] - message = f"""\ -Please provide a self-contained Python script that solves the following problem in a markdown code block: -{prompt.strip()} -""" - - for _ in range(num_samples): - while True: - try: - response = model.generate_content( - message, - generation_config=genai_config - ) - output = response.candidates[0].content.parts[0].text - outputs.append(output) - break - except Exception as e: - if "list index out of range" in str(e): - # append dummy response - outputs.append("NO_RESPONSE") - break - else: - print(e) - continue - - all_outputs.append(outputs) - - return all_outputs - - -def make_model( - model: str, - backend: str, - subset: str, - split: str, - dataset: str = "bigcodebench", - temperature: float = 0.0, - tp=1, - direct_completion=False, - base_url=None, - trust_remote_code=False, - tokenizer_name=None, - tokenizer_legacy=True, -): - if backend == "vllm": - return GeneralVllmDecoder( - name=model, - subset=subset, - split=split, - temperature=temperature, - dataset=dataset, - tp=tp, - direct_completion=direct_completion, - trust_remote_code=trust_remote_code, - tokenizer_name=tokenizer_name, - tokenizer_legacy=tokenizer_legacy, - ) - elif backend == "hf": - return GenenralHfTorchDecoder( - name=model, - subset=subset, - split=split, - temperature=temperature, - dataset=dataset, - direct_completion=direct_completion, - trust_remote_code=trust_remote_code, - tokenizer_name=tokenizer_name, - tokenizer_legacy=tokenizer_legacy, - ) - elif backend == "openai": - return OpenAIChatDecoder( - name=model, - subset=subset, - split=split, - temperature=temperature, - base_url=base_url, - ) - elif backend == "mistral": - return MistralChatDecoder( - name=model, - subset=subset, - split=split, - temperature=temperature, - ) - elif backend == "anthropic": - return AnthropicMessageDecoder( - name=model, - subset=subset, - split=split, - temperature=temperature, - ) - elif backend == "google": - return GeminiDecoder( - name=model, - subset=subset, - split=split, - temperature=temperature, - ) \ No newline at end of file diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py new file mode 100644 index 0000000..2203196 --- /dev/null +++ b/bigcodebench/provider/__init__.py @@ -0,0 +1,100 @@ +from bigcodebench.provider.base import DecoderBase + + +def make_model( + model: str, + backend: str, + subset: str, + split: str, + dataset: str = "bigcodebench", + temperature: float = 0.0, + # instruction model only + instruction_prefix=None, + response_prefix=None, + # vllm only + tp=1, + direct_completion=False, + base_url=None, + trust_remote_code=False, + # hf only + attn_implementation="eager", + # tokenizer + tokenizer_name=None, + tokenizer_kwargs=None, +) -> DecoderBase: + if backend == "vllm": + from bigcodebench.provider.vllm import VllmDecoder + + return VllmDecoder( + name=model, + subset=subset, + split=split, + temperature=temperature, + dataset=dataset, + direct_completion=direct_completion, + tensor_parallel_size=tp, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "hf": + from bigcodebench.provider.hf import HuggingFaceDecoder + + return HuggingFaceDecoder( + name=model, + subset=subset, + split=split, + temperature=temperature, + dataset=dataset, + direct_completion=direct_completion, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + attn_implementation=attn_implementation, + ) + elif backend == "openai": + from bigcodebench.provider.openai import OpenAIChatDecoder + + assert not direct_completion, f"{backend} backend does not serve base model" + return OpenAIChatDecoder( + name=model, + subset=subset, + split=split, + temperature=temperature, + base_url=base_url, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "mistral": + from bigcodebench.provider.mistral import MistralChatDecoder + + return MistralChatDecoder( + name=model, + subset=subset, + split=split, + temperature=temperature, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "anthropic": + from bigcodebench.provider.anthropic import AnthropicDecoder + + assert not direct_completion, f"{backend} backend does not serve base model" + return AnthropicDecoder( + name=model, + subset=subset, + split=split, + temperature=temperature, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "google": + from bigcodebench.provider.google import GoogleDecoder + + assert not direct_completion, f"{backend} backend does not serve base model" + return GoogleDecoder( + name=model, + subset=subset, + split=split, + temperature=temperature, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) \ No newline at end of file diff --git a/bigcodebench/provider/anthropic.py b/bigcodebench/provider/anthropic.py new file mode 100644 index 0000000..3f51ecf --- /dev/null +++ b/bigcodebench/provider/anthropic.py @@ -0,0 +1,54 @@ +import os +from typing import List + +import anthropic + +from bigcodebench.gen.util import anthropic_request +from bigcodebench.provider.base import DecoderBase +from bigcodebench.provider.utility import make_raw_chat_prompt + +class AnthropicDecoder(DecoderBase): + def __init__(self, name: str, **kwargs) -> None: + super().__init__(name, **kwargs) + self.client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_KEY")) + + def codegen( + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + if do_sample: + assert self.temperature > 0, "Temperature must be positive for sampling" + + if not do_sample: + assert batch_size == 1, "Sampling only supports batch size of 1" + + all_outputs = [] + for prompt in tqdm(prompts): + outputs = [] + + for _ in range(num_samples): + message = anthropic_request.make_auto_request( + client=self.client, + model=self.name, + messages=[ + { + "role": "user", + "content": make_raw_chat_prompt( + task_prompt=prompt, + subset=self.subset, + split=self.split, + instruction_prefix=self.instruction_prefix, + response_prefix=self.response_prefix, + tokenizer=None, + ) + } + ], + max_tokens=self.max_new_tokens, + temperature=self.temperature, + stop_sequences=self.eos, + ) + outputs.append(message.content[0].text) + all_outputs.append(outputs) + return outputs + + def is_direct_completion(self) -> bool: + return False \ No newline at end of file diff --git a/bigcodebench/provider/base.py b/bigcodebench/provider/base.py new file mode 100644 index 0000000..e5bad6d --- /dev/null +++ b/bigcodebench/provider/base.py @@ -0,0 +1,49 @@ +from abc import ABC, abstractmethod +from typing import List + +from evalplus.provider.utility import EOS + + +class DecoderBase(ABC): + def __init__( + self, + name: str, + subset: str, + split: str, + temperature: float = 0.8, + max_new_tokens: int = 5120, + dtype: str = "bfloat16", # default + direct_completion: bool = False, + trust_remote_code: bool = False, + tokenizer_name: str = None, + tokenizer_legacy: bool = False, + ) -> None: + print("Initializing a decoder model: {} ...".format(name)) + self.name = name + self.subset = subset + self.split = split + self.temperature = temperature + self.eos = EOS + self.skip_special_tokens = False + self.max_new_tokens = max_new_tokens + self.dtype = dtype + self.direct_completion = direct_completion + self.trust_remote_code = trust_remote_code + self.tokenizer_name = tokenizer_name + self.tokenizer_legacy = tokenizer_legacy + + @abstractmethod + def codegen( + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + pass + + @abstractmethod + def is_direct_completion(self) -> bool: + pass + + def __repr__(self) -> str: + return self.name + + def __str__(self) -> str: + return self.name \ No newline at end of file diff --git a/bigcodebench/provider/google.py b/bigcodebench/provider/google.py new file mode 100644 index 0000000..07237c1 --- /dev/null +++ b/bigcodebench/provider/google.py @@ -0,0 +1,58 @@ +import os +from typing import List + +import google.generativeai as genai + + +from bigcodebench.provider.base import DecoderBase +from bigcodebench.gen.util.google_request import make_auto_request +from bigcodebench.provider.utility import make_raw_chat_prompt + + +class GoogleDecoder(DecoderBase): + def __init__(self, name: str, **kwargs): + super().__init__(name, **kwargs) + genai.configure(api_key=os.getenv("GOOGLE_API_KEY")) + self.client = genai.GenerativeModel(name) + + def codegen( + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + if do_sample: + assert self.temperature > 0, "Temperature must be positive for sampling" + + all_outputs = [] + + for prompt in tqdm(prompts): + ret_texts = [] + message = make_raw_chat_prompt( + task_prompt=prompt, + subset=self.subset, + split=self.split, + instruction_prefix=self.instruction_prefix, + response_prefix=self.response_prefix, + tokenizer=None, + ) + replies = make_auto_request( + self.client, + message, + self.name, + n=batch_size, + max_tokens=self.max_new_tokens, + temperature=self.temperature, + ) + for candidate in replies.candidates: + parts = candidate.content.parts + if parts: + ret_texts.append(parts[0].text) + else: + print("Empty response!") + ret_texts.append("") + print(f"{candidate.safety_ratings = }") + ret_texts.append("") + all_outputs.append(ret_texts + [""] * (batch_size - len(ret_texts))) + + return all_outputs + + def is_direct_completion(self) -> bool: + return False \ No newline at end of file diff --git a/bigcodebench/provider/hf.py b/bigcodebench/provider/hf.py new file mode 100644 index 0000000..b260360 --- /dev/null +++ b/bigcodebench/provider/hf.py @@ -0,0 +1,95 @@ +from typing import List + +import torch +from stop_sequencer import StopSequencer +from transformers import AutoModelForCausalLM, AutoTokenizer + +from evalplus.provider.base import DecoderBase +from evalplus.provider.utility import ( + extra_eos_for_direct_completion, + make_raw_chat_prompt, +) + + +class HuggingFaceDecoder(DecoderBase): + def __init__( + self, + name: str, + dataset: str, + attn_implementation: str = "eager", + **kwargs, + ): + super().__init__(name=name, **kwargs) + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + kwargs = { + "device_map": "auto", + "trust_remote_code": self.trust_remote_code, + "torch_dtype": getattr(torch, self.dtype), + "attn_implementation": attn_implementation, # "eager", "flash_attention_2", "sdpa" + } + self.skip_special_tokens = True + + print(f"{kwargs = }") + + self.tokenizer = AutoTokenizer.from_pretrained(name, use_fast=False, legacy=self.tokenizer_legacy) + if self.is_direct_completion(): # no chat template + self.eos += extra_eos_for_direct_completion(dataset) + else: # with chat template + self.eos += ["\n```\n"] + + print(f"{self.eos = }") + self.model = AutoModelForCausalLM.from_pretrained(name, **kwargs) + self.model = self.model.to(self.device) + + def is_direct_completion(self) -> bool: + return self.direct_completion or self.tokenizer.chat_template is None + + @torch.inference_mode() + def codegen( + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + if self.temperature == 0: + assert not do_sample + assert num_samples == 1 + + prompts = [ + prompt + if self.is_direct_completion() + else make_raw_chat_prompt( + prompt, self.subset, self.split, self.instruction_prefix, self.response_prefix, self.tokenizer, self.direct_completion + ) + for prompt in prompts + ] + input_tokens = self.tokenizer.encode(prompts, return_tensors="pt").to( + self.device + ) + kwargs = {} + if do_sample: + kwargs["top_p"] = 0.95 + kwargs["temperature"] = self.temperature + + outputs = self.model.generate( + input_tokens, + max_new_tokens=self.max_new_tokens, + do_sample=do_sample, + num_return_sequences=num_samples, + pad_token_id=self.tokenizer.eos_token_id, + stop_strings=self.eos, + tokenizer=self.tokenizer, + **kwargs, + ) + + gen_strs = self.tokenizer.batch_decode( + outputs[:, input_tokens.size(-1) :], + skip_special_tokens=self.skip_special_tokens, + ) + outputs = [] + # removes eos tokens. + for output in gen_strs: + min_index = 10000 + for eos in self.eos: + if eos in output: + min_index = min(min_index, output.index(eos)) + outputs.append(output[:min_index].replace("\t", " ")) + return outputs \ No newline at end of file diff --git a/bigcodebench/provider/mistral.py b/bigcodebench/provider/mistral.py new file mode 100644 index 0000000..c016e2e --- /dev/null +++ b/bigcodebench/provider/mistral.py @@ -0,0 +1,50 @@ +import os +from typing import List + +import anthropic + +from bigcodebench.provider.base import DecoderBase +from bigcodebench.provider.utility import make_raw_chat_prompt + +class MistralDecoder(DecoderBase): + def __init__(self, name: str, **kwargs) -> None: + super().__init__(name, **kwargs) + self.client = mistral.Mistral(api_key=os.getenv("MISTRAL_API_KEY")) + + def codegen( + self, prompt: str, do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + if do_sample: + assert self.temperature > 0, "Temperature must be positive for sampling" + + all_outputs = [] + for prompt in tqdm(prompts): + outputs = [] + + for _ in range(num_samples): + message = mistral_request.make_auto_request( + client=self.client, + model=self.name, + messages=[ + ChatMessage( + role="user", + content=make_raw_chat_prompt( + task_prompt=prompt, + subset=self.subset, + split=self.split, + instruction_prefix=self.instruction_prefix, + response_prefix=self.response_prefix, + tokenizer=None, + direct_completion=None, + ) + ) + ], + max_tokens=self.max_new_tokens, + **kwargs, + ) + outputs.append(message.content[0].text) + all_outputs.append(outputs) + return all_outputs + + def is_direct_completion(self) -> bool: + return False \ No newline at end of file diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py new file mode 100644 index 0000000..3f55659 --- /dev/null +++ b/bigcodebench/provider/openai.py @@ -0,0 +1,46 @@ +import os +from typing import List + +import openai + +from evalplus.gen.util import openai_request +from evalplus.provider.base import DecoderBase +from bigcodebench.provider.utility import make_raw_chat_prompt + +class OpenAIChatDecoder(DecoderBase): + def __init__(self, name: str, base_url=None, **kwargs) -> None: + super().__init__(name, **kwargs) + self.client = openai.OpenAI( + api_key=os.getenv("OPENAI_API_KEY", "none"), base_url=base_url + ) + + def codegen( + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + if do_sample: + assert self.temperature > 0, "Temperature must be positive for sampling" + all_outputs = [] + for prompt in tqdm(prompts): + message = make_raw_chat_prompt( + task_prompt=prompt, + subset=self.subset, + split=self.split, + instruction_prefix=self.instruction_prefix, + response_prefix=self.response_prefix, + tokenizer=None, + ) + ret = openai_request.make_auto_request( + self.client, + message=message, + model=self.name, + max_tokens=self.max_new_tokens, + temperature=self.temperature, + n=num_samples, + ) + for item in ret.choices: + outputs.append(item.message.content) + all_outputs.append(outputs) + return all_outputs + + def is_direct_completion(self) -> bool: + return False \ No newline at end of file diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py new file mode 100644 index 0000000..93a739b --- /dev/null +++ b/bigcodebench/provider/utility.py @@ -0,0 +1,65 @@ +from typing import List + +EOS = [ + "<|endoftext|>", + "<|endofmask|>", + "", + "\nif __name__", + "\ndef main(", + "\nprint(", +] + + +def extra_eos_for_direct_completion(dataset) -> List[str]: + if dataset.lower() == "bigcodebench": + return ["\ndef ", "\nclass ", "\nimport ", "\nfrom ", "\nassert "] + raise ValueError(f"Unknown dataset: {dataset}") + + +# some random words which serves as the splitter +_MAGIC_SPLITTER_ = "-[[]]-this-is-really-our-highest-priority-[[]]-" + + +def make_raw_chat_prompt( + task_prompt: str, + subset: str, + split: str, + instruction_prefix: str, + response_prefix: str, + tokenizer: AutoTokenizer, + direct_completion: bool = False, +) -> str: + # directly return prompt if it does not have a tokenizer.chat_template + if tokenizer: + if tokenizer.chat_template is None or direct_completion: + return task_prompt + + assert instruction_prefix is not None, "Instruction prefix is required!" + assert response_prefix is not None, "Response prefix is required!" + + if split == "complete": + task_prompt = f"""\ +{instruction_prefix} +``` +{task_prompt.strip()} +``` +""" + else: + task_prompt = f"""\ +{instruction_prefix} +{task_prompt.strip()} +""" + response = f"""\ +{response_prefix} +```python +{_MAGIC_SPLITTER_} +``` +""" + task_prompt = tokenizer.apply_chat_template( + [ + {"role": "user", "content": task_prompt}, + {"role": "assistant", "content": response}, + ], + tokenize=False, + ).split(_MAGIC_SPLITTER_)[0] + return task_prompt \ No newline at end of file diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py new file mode 100644 index 0000000..178b3b6 --- /dev/null +++ b/bigcodebench/provider/vllm.py @@ -0,0 +1,67 @@ +from typing import List + +from transformers import AutoTokenizer +from vllm import LLM, SamplingParams + +from bigcodebench.provider.base import DecoderBase +from bigcodebench.provider.utility import ( + extra_eos_for_direct_completion, + make_raw_chat_prompt, +) + +class VllmDecoder(DecoderBase): + def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: + super().__init__(name, **kwargs) + + kwargs = { + "tensor_parallel_size": int(os.getenv("VLLM_N_GPUS", tp)), + "dtype": self.dtype, + "trust_remote_code": self.trust_remote_code, + } + if self.tokenizer_name is None: + self.tokenizer_name = self.name + + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=self.tokenizer_legacy) + if self.is_direct_completion(): + self.eos += extra_eos_for_direct_completion(dataset) + else: + self.eos += ["\n```\n"] + self.llm = LLM(model=name, max_model_len=self.max_new_tokens, **kwargs) + self.llm.set_tokenizer(tokenizer=self.tokenizer) + + def is_direct_completion(self) -> bool: + return self.tokenizer.chat_template is None or self.direct_completion + + def codegen( + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + if do_sample: + assert self.temperature > 0, "Temperature must be greater than 0!" + + prompts = [ + make_raw_chat_prompt( + task_prompt=prompt, + subset=self.subset, + split=self.split, + instruction_prefix=self.instruction_prefix, + response_prefix=self.response_prefix, + tokenizer=self.tokenizer, + direct_completion=self.direct_completion, + ) + for prompt in prompts + ] + vllm_outputs = self.llm.generate( + prompts, + SamplingParams( + n=num_samples, + temperature=self.temperature, + max_tokens=self.max_new_tokens, + top_p=0.95 if do_sample else 1.0, + stop=self.eos, + skip_special_tokens=self.skip_special_tokens, + ), + use_tqdm=True, + ) + + gen_strs = [[x.text.replace("\t", " ") for x in output.outputs] for output in vllm_outputs] + return gen_strs \ No newline at end of file From db22671b631016d8ae76f4a06dbced24a08a59b7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 06:02:14 +0800 Subject: [PATCH 152/325] refactor: use model provide for gen --- bigcodebench/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index d1f22d2..a110e90 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -3,7 +3,7 @@ import argparse from typing import Optional, Tuple -from bigcodebench.model import DecoderBase, make_model +from bigcodebench.provider import DecoderBase, make_model from bigcodebench.data import get_bigcodebench, write_jsonl from bigcodebench.sanitize import sanitize from rich.progress import ( From ae94984ddd7e791fd32e10925cd9957024ed610d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 06:02:58 +0800 Subject: [PATCH 153/325] doc: 0.2.0 pre-release --- ADVANCED_USAGE.md | 301 +++++++++++++++++++++++++++++++++++ README.md | 332 +++++++-------------------------------- _README.md | 393 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 750 insertions(+), 276 deletions(-) create mode 100755 ADVANCED_USAGE.md create mode 100755 _README.md diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md new file mode 100755 index 0000000..526031b --- /dev/null +++ b/ADVANCED_USAGE.md @@ -0,0 +1,301 @@ +## 🔥 Advanced Start + +To get started, please first set up the environment: + +```bash +# Install to use bigcodebench.evaluate +pip install bigcodebench --upgrade +# If you want to use the evaluate locally, you need to install the requirements +pip install -I -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt + +# Install to use bigcodebench.generate +# You are strongly recommended to install the generate dependencies in a separate environment +pip install bigcodebench[generate] --upgrade +``` + +
⏬ Install nightly version :: click to expand :: +
+ +```bash +# Install to use bigcodebench.evaluate +pip install "git+https://github.com/bigcode-project/bigcodebench.git" --upgrade + +# Install to use bigcodebench.generate +pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcodebench[generate]" --upgrade +``` + +
+
+ +
⏬ Using BigCodeBench as a local repo? :: click to expand :: +
+ +```bash +git clone https://github.com/bigcode-project/bigcodebench.git +cd bigcodebench +export PYTHONPATH=$PYTHONPATH:$(pwd) +# Install to use bigcodebench.evaluate +pip install -e . +# Install to use bigcodebench.generate +pip install -e .[generate] +``` + +
+
+ +### 🚀 Local Generation + +```bash +# when greedy, there is no need for temperature and n_samples +bigcodebench.generate \ + --model [model_name] \ + --split [complete|instruct] \ + --subset [full|hard] \ + [--greedy] \ + --bs [bs] \ + --temperature [temp] \ + --n_samples [n_samples] \ + --resume \ + --backend [vllm|openai|mistral|anthropic|google|hf] \ + --tp [TENSOR_PARALLEL_SIZE] \ + [--trust_remote_code] \ + [--base_url [base_url]] \ + [--tokenizer_name [tokenizer_name]] +``` +> +The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: +> +```bash +# If you are using GPUs +docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ + --model [model_name] \ + --split [complete|instruct] \ + --subset [full|hard] \ + [--greedy] \ + --bs [bs] \ + --temperature [temp] \ + --n_samples [n_samples] \ + --resume \ + --backend [vllm|openai|mistral|anthropic|google|hf] \ + --tp [TENSOR_PARALLEL_SIZE] + +# ...Or if you are using CPUs +docker run -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ + --model [model_name] \ + --split [complete|instruct] \ + --subset [full|hard] \ + [--greedy] \ + --bs [bs] \ + --temperature [temp] \ + --n_samples [n_samples] \ + --resume \ + --backend [vllm|hf|openai|mistral|anthropic|google] +``` +> +```bash +# If you wish to use gated or private HuggingFace models and datasets +docker run -e HUGGING_FACE_HUB_TOKEN=$token -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments4 + +# Similarly, to use other backends that require authentication +docker run -e OPENAI_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments +docker run -e ANTHROPIC_KEY=$ANTHROPIC_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments +docker run -e MISTRAL_KEY=$MISTRAL_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments +docker run -e GOOGLE_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments +``` +> +Following which, you can run the built container as shown in above. +> +
🤔 Structure of `problem`? :: click to expand :: +
+ +* `task_id` is the identifier string for the task +* `entry_point` is the name of the function +* `complete_prompt` is the prompt for BigCodeBench-Complete +* `instruct_prompt` is the prompt for BigCodeBench-Instruct ++ `canonical_solution` is the ground-truth implementation ++ `test` is the `unittest.TestCase` class + +
+
+ +> [!Note] +> +> **Expected Schema of `[model_name]--bigcodebench-[task]--[backend]-[temp]-[n_samples].jsonl`** +> +> 1. `task_id`: Task ID, which are the keys of `get_bigcodebench()` +> 2. `solution` (optional): Self-contained solution (usually including the prompt) +> 3. `raw_solution` (optional): The raw solution generated by the LLM +> * Example: `{"task_id": "BigCodeBench/?", "solution": "def f():\n return 1", "raw_solution": "def f():\n return 1\nprint(f())"}` + + +
🔎 Checking the compatibility of post-processed code:: click to expand :: +
+ +To double-check the post-processing results, you can use `bigcodebench.syncheck` to check the code validity before and after sanitization, which will print erroneous code snippets and why they are wrong: + +```bash +# 💡 If you are storing codes in jsonl: +bigcodebench.syncheck --samples samples.jsonl + +# 💡 If you are storing codes in directories: +bigcodebench.syncheck --samples /path/to/vicuna-[??]b_temp_[??] + +# 💡 Or change the entrypoint to bigcodebench.syncheck in any pre-built docker image, like +docker run -it --entrypoint bigcodebench.syncheck -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --samples samples.jsonl +``` + +
+
+ + +### Local Evaluation + +You are strongly recommended to use a sandbox such as [docker](https://docs.docker.com/get-docker/): + +```bash +# Mount the current directory to the container +# If you want to change the RAM address space limit (in MB, 30 GB by default): `--max-as-limit XXX` +# If you want to change the RAM data segment limit (in MB, 30 GB by default): `--max-data-limit` +# If you want to change the RAM stack limit (in MB, 10 MB by default): `--max-stack-limit` +# If you want to increase the execution time limit (in seconds, 240 seconds by default): `--min-time-limit` +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl + +# If you only want to check the ground truths +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --check-gt-only +``` + +...Or if you want to try it locally regardless of the risks ⚠️: + +First, install the dependencies for BigCodeBench: + +```bash +pip install -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt +``` + +Then, run the evaluation: + +```bash +# ...Or locally ⚠️ +bigcodebench.evaluate --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl +# ...If you really don't want to check the ground truths +bigcodebench.evaluate --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt +# If you want to save the pass rate to a file +bigcodebench.evaluate --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --save_pass_rate + +# You are strongly recommended to use the following command to clean up the environment after evaluation: +pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; +rm -rf /tmp/* +``` + +> [!Tip] +> +> If you want to customize the `k` in `Pass@k`, please pass `--pass_k` with a comma-separated string. +> For example, if you want to use `Pass@1` and `Pass@100`, you can pass `--pass_k 1,100`. + +> [!Tip] +> +> Do you use a very slow machine? +> +> LLM solutions are regarded as **failed** on timeout (and OOM etc.). +> Specifically, we set the dynamic timeout based on the ground-truth solution's runtime. +> +> Additionally, you are **NOT** encouraged to make your test-bed over stressed while running evaluation. +> For example, using `--parallel 64` on a 4-core machine or doing something else during evaluation are bad ideas... + +
⌨️ More command-line flags :: click to expand :: +
+ +* `--parallel`: by default half of the cores + +
+
+ +The output should be like (below is GPT-4 greedy decoding example): + +``` +Asserting the groundtruth... +Expected outputs computed in 1200.0 seconds +Reading samples... +1140it [00:00, 1901.64it/s] +Evaluating samples... +100%|██████████████████████████████████████████| 1140/1140 [19:53<00:00, 6.75it/s] +BigCodeBench-Instruct-calibrated +Groundtruth pass rate: 1.000 +pass@1: 0.568 +``` + +- A cache file named like `samples_eval_results.json` will be cached. Remove it to re-run the evaluation + +
🤔 How long it would take? :: click to expand :: +
+ +If you do greedy decoding where there is only one sample for each task, the evaluation should take just a few minutes on Intel(R) Xeon(R) Gold 6150 CPU @ 2.70GHz, composed of 2 sockets, with 18 cores per socket. However, if you have multiple samples for each task, the evaluation will take longer. +Here are some tips to speed up the evaluation: + +* Use `--parallel $(nproc)` +* Use our pre-evaluated results (see [LLM-generated code](#-LLM-generated-code)) + +
+
+ +## 🔍 Failure Inspection + +You can inspect the failed samples by using the following command: + +```bash +# Inspect the failed samples and save the results to `inspect/` +bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard + +# Re-run the inspection in place +bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard --in_place +``` + +## 🚀 Full Script + +We provide a sample script to run the full pipeline: + +```bash +bash run.sh +``` + +## 📊 Result Analysis + +We provide a script to replicate the analysis like Elo Rating and Task Solve Rate, which helps you understand the performance of the models further. + +```bash +To run the analysis, you need to put all the `samples_eval_results.json` files in a `results` folder, which is in the same directory as the script. + +```bash +cd analysis +python get_results.py +``` + +## 💻 LLM-generated Code + +We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): +* See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. + +## 🐞 Known Issues + +- [x] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizers may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. + +- [x] Due to the flakiness in the evaluation, the execution results may vary slightly (~0.2% for Full set, and ~0.6% for Hard set) between runs. We are working on improving the evaluation stability. + +- [x] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. If the issue persists ,please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). + +- [x] We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code. + +## 📜 Citation + +```bibtex +@article{zhuo2024bigcodebench, + title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}, + author={Zhuo, Terry Yue and Vu, Minh Chien and Chim, Jenny and Hu, Han and Yu, Wenhao and Widyasari, Ratnadira and Yusuf, Imam Nur Bani and Zhan, Haolan and He, Junda and Paul, Indraneil and others}, + journal={arXiv preprint arXiv:2406.15877}, + year={2024} +} +``` + +## 🙏 Acknowledgement + +- [EvalPlus](https://github.com/evalplus/evalplus) diff --git a/README.md b/README.md index 5bb72ba..f72ea2a 100755 --- a/README.md +++ b/README.md @@ -16,26 +16,32 @@

- 🌸About • - 🔥Quick Start • - 🔍Failure Inspection • - 🚀Full Script • - 📊Result Analysis • - 💻LLM-generated Code • - 🐞Known Issues • - 📜Citation • - 🙏Acknowledgement + 📰 News • + 🔥 Quick Start • + 🚀 Remote Evaluation • + 💻 LLM-generated Code • + 📜 Advanced Usage • + 🙏 Acknowledgement

-## News +## 📰 News +- **[2024-10-05]** We create a public code execution API on the Hugging Face space. +- **[2024-10-01]** We have evaluated 139 models on BigCodeBench-Hard so far. Take a look at the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard)! - **[2024-08-19]** To make the evaluation fully reproducible, we add a real-time code execution session to the leaderboard. It can be viewed [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). - **[2024-08-02]** We release `bigcodebench==v0.1.9`. + +
More News :: click to expand :: +
+ - **[2024-07-18]** We announce a subset of BigCodeBench, BigCodeBench-Hard, which includes 148 tasks that are more aligned with the real-world programming tasks. The details are available [in this blog post](https://huggingface.co/blog/terryyz/bigcodebench-hard). The dataset is available [here](https://huggingface.co/datasets/bigcode/bigcodebench-hard). The new release is `bigcodebench==v0.1.8`. - **[2024-06-28]** We release `bigcodebench==v0.1.7`. - **[2024-06-27]** We release `bigcodebench==v0.1.6`. - **[2024-06-19]** We start the Hugging Face BigCodeBench Leaderboard! The leaderboard is available [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). - **[2024-06-18]** We release BigCodeBench, a new benchmark for code generation with 1140 software-engineering-oriented programming tasks. Preprint is available [here](https://arxiv.org/abs/2406.15877). PyPI package is available [here](https://pypi.org/project/bigcodebench/) with the version `0.1.5`. +
+
+ ## 🌸 About ### BigCodeBench @@ -45,7 +51,7 @@ To facilitate the evaluation of LLMs on BigCodeBench, we provide this Python pac ### Why BigCodeBench? -BigCodeBench focuses on the evaluation of LLM4Code with *diverse function calls* and *complex instruction*, with: +BigCodeBench focuses on task automation via code generation with *diverse function calls* and *complex instructions*, with: * ✨ **Precise evaluation & ranking**: See [our leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard) for latest LLM rankings before & after rigorous evaluation. * ✨ **Pre-generated samples**: BigCodeBench accelerates code intelligence research by open-sourcing [LLM-generated samples](#-LLM-generated-code) for various models -- no need to re-run the expensive benchmarks! @@ -60,297 +66,77 @@ BigCodeBench focuses on the evaluation of LLM4Code with *diverse function calls* To get started, please first set up the environment: ```bash -# Install to use bigcodebench.evaluate -pip install bigcodebench --upgrade -# If you want to use the evaluate locally, you need to install the requirements -pip install -I -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt - -# Install to use bigcodebench.generate -# You are strongly recommended to install the generate dependencies in a separate environment +# By default, you will use the remote evaluation API to execute the output samples. pip install bigcodebench[generate] --upgrade -``` -
⏬ Install nightly version :: click to expand :: -
- -```bash -# Install to use bigcodebench.evaluate -pip install "git+https://github.com/bigcode-project/bigcodebench.git" --upgrade +# You are suggested to use `flash-attn` for generating code samples. +pip install packaging ninja +pip install flash-attn --no-build-isolation +# Note: if you have installation problem, consider using pre-built +# wheels from https://github.com/Dao-AILab/flash-attention/releases ``` -
-
- -
⏬ Using BigCodeBench as a local repo? :: click to expand :: +
⏬ Install nightly version :: click to expand ::
```bash -git clone https://github.com/bigcode-project/bigcodebench.git -cd bigcodebench -export PYTHONPATH=$PYTHONPATH:$(pwd) -# Install to use bigcodebench.evaluate -pip install -e . # Install to use bigcodebench.generate -pip install -e .[generate] +pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcodebench[generate]" --upgrade ```
-### Code Generation -You are suggested to use `flash-attn` for generating code samples. -```bash -pip install -U flash-attn -``` +## 🚀 Remote Evaluation + +We use the greedy decoding as an example to show how to evaluate the generated code samples via remote API. -To generate code samples from a model, you can use the following command: -> -```bash -# when greedy, there is no need for temperature and n_samples -bigcodebench.generate \ - --model [model_name] \ - --split [complete|instruct] \ - --subset [full|hard] \ - [--greedy] \ - --bs [bs] \ - --temperature [temp] \ - --n_samples [n_samples] \ - --resume \ - --backend [vllm|hf|openai|mistral|anthropic|google] \ - --tp [gpu_number] \ - [--trust_remote_code] \ - [--base_url [base_url]] \ - [--tokenizer_name [tokenizer_name]] -``` -> -The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: -> ```bash -# If you are using GPUs -docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ - --model [model_name] \ - --split [complete|instruct] \ - --subset [full|hard] \ - [--greedy] \ - --bs [bs] \ - --temperature [temp] \ - --n_samples [n_samples] \ - --resume \ - --backend [vllm|hf|openai|mistral|anthropic|google] \ - --tp [gpu_number] - -# ...Or if you are using CPUs -docker run -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ - --model [model_name] \ - --split [complete|instruct] \ - --subset [full|hard] \ - [--greedy] \ - --bs [bs] \ - --temperature [temp] \ - --n_samples [n_samples] \ - --resume \ - --backend [vllm|hf|openai|mistral|anthropic|google] +bigcodebench.evaluate \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --split [complete|instruct] \ + --subset [full|hard] \ + --backend [vllm|openai|anthropic|google|mistral|hf] \ + --tp [TENSOR_PARALLEL_SIZE] \ + --greedy ``` -> -```bash -# If you wish to use gated or private HuggingFace models and datasets -docker run -e HUGGING_FACE_HUB_TOKEN=$token -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments4 +- The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated.jsonl`. +- The evaluation results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_eval_results.json`. +- The pass@k results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_pass_at_k.json`. -# Similarly, to use other backends that require authentication -docker run -e OPENAI_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments -docker run -e GOOGLE_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments -docker run -e ANTHROPIC_KEY=$ANTHROPIC_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments -``` -> -Following which, you can run the built container as shown in above. +> [!Note] > -
🤔 Structure of `problem`? :: click to expand :: -
- -* `task_id` is the identifier string for the task -* `entry_point` is the name of the function -* `complete_prompt` is the prompt for BigCodeBench-Complete -* `instruct_prompt` is the prompt for BigCodeBench-Instruct -+ `canonical_solution` is the ground-truth implementation -+ `test` is the `unittest.TestCase` class - -
-
+> Remotely executing on BigCodeBench-Full typically takes 5-7 minutes, and on BigCodeBench-Hard typically takes 3-5 minutes. > [!Note] > -> **Expected Schema of `[model_name]--bigcodebench-[task]--[backend]-[temp]-[n_samples].jsonl`** +> BigCodeBench uses different prompts for base and chat models. +> By default it is detected by `tokenizer.chat_template` when using `hf`/`vllm` as backend. +> For other backends, only chat mode is allowed. > -> 1. `task_id`: Task ID, which are the keys of `get_bigcodebench()` -> 2. `solution` (optional): Self-contained solution (usually including the prompt) -> * Example: `{"task_id": "BigCodeBench/?", "solution": "def f():\n return 1"}` - -### Code Post-processing - -LLM-generated text may not be compilable code for including natural language lines or incomplete extra code. -We provide a tool namely `bigcodebench.sanitize` to clean up the code: +> Therefore, if your base models come with a `tokenizer.chat_template`, +> please add `--direct_completion` to avoid being evaluated +> in a chat mode. +Access OpenAI APIs from [OpenAI Console](https://platform.openai.com/) ```bash -# 💡 If you want to get the calibrated results: -bigcodebench.sanitize --samples samples.jsonl --calibrate -# Sanitized code will be produced to `samples-sanitized-calibrated.jsonl` - -# 💡 Optionally run the sanitization step with multiprocessing to speedup -bigcodebench.sanitize --samples samples.jsonl --calibrate --parallel 8 - -# 💡 If you want to get the original results: -bigcodebench.sanitize --samples samples.jsonl -# Sanitized code will be produced to `samples-sanitized.jsonl` - -# 💡 If you are storing codes in directories: -bigcodebench.sanitize --samples /path/to/vicuna-[??]b_temp_[??] -# Sanitized code will be produced to `/path/to/vicuna-[??]b_temp_[??]-sanitized` +export OPENAI_API_KEY= ``` -If you want to use the pre-built docker images for post-processing, you can use the following command: - +Access Anthropic APIs from [Anthropic Console](https://console.anthropic.com/) ```bash -# Change the entrypoint to bigcodebench.sanitize in any pre-built docker image, like bigcodebench/bigcodebench-evaluate:latest -docker run -it --entrypoint bigcodebench.sanitize -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --samples samples.jsonl +export ANTHROPIC_API_KEY= ``` -
🔎 Checking the compatibility of post-processed code:: click to expand :: -
- -To double-check the post-processing results, you can use `bigcodebench.syncheck` to check the code validity before and after sanitization, which will print erroneous code snippets and why they are wrong: - +Access Mistral APIs from [Mistral Console](https://console.mistral.ai/) ```bash -# 💡 If you are storing codes in jsonl: -bigcodebench.syncheck --samples samples.jsonl - -# 💡 If you are storing codes in directories: -bigcodebench.syncheck --samples /path/to/vicuna-[??]b_temp_[??] - -# 💡 Or change the entrypoint to bigcodebench.syncheck in any pre-built docker image, like -docker run -it --entrypoint bigcodebench.syncheck -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --samples samples.jsonl +export MISTRAL_API_KEY= ``` -
-
- - -### Code Evaluation - -You are strongly recommended to use a sandbox such as [docker](https://docs.docker.com/get-docker/): - +Access Gemini APIs from [Google AI Studio](https://aistudio.google.com/) ```bash -# Mount the current directory to the container -# If you want to change the RAM address space limit (in MB, 30 GB by default): `--max-as-limit XXX` -# If you want to change the RAM data segment limit (in MB, 30 GB by default): `--max-data-limit` -# If you want to change the RAM stack limit (in MB, 10 MB by default): `--max-stack-limit` -# If you want to increase the execution time limit (in seconds, 240 seconds by default): `--min-time-limit` -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl - -# If you only want to check the ground truths -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --check-gt-only -``` - -...Or if you want to try it locally regardless of the risks ⚠️: - -First, install the dependencies for BigCodeBench: - -```bash -pip install -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt -``` - -Then, run the evaluation: - -```bash -# ...Or locally ⚠️ -bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl -# ...If you really don't want to check the ground truths -bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt -# If you want to save the pass rate to a file -bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --save_pass_rate - -# You are strongly recommended to use the following command to clean up the environment after evaluation: -pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; -rm -rf /tmp/* -``` - -> [!Tip] -> -> Do you use a very slow machine? -> -> LLM solutions are regarded as **failed** on timeout (and OOM etc.). -> Specifically, we set the dynamic timeout based on the ground-truth solution's runtime. -> -> Additionally, you are **NOT** encouraged to make your test-bed over stressed while running evaluation. -> For example, using `--parallel 64` on a 4-core machine or doing something else during evaluation are bad ideas... - -
⌨️ More command-line flags :: click to expand :: -
- -* `--parallel`: by default half of the cores - -
-
- -The output should be like (below is GPT-4 greedy decoding example): - -``` -Asserting the groundtruth... -Expected outputs computed in 1200.0 seconds -Reading samples... -1140it [00:00, 1901.64it/s] -Evaluating samples... -100%|██████████████████████████████████████████| 1140/1140 [19:53<00:00, 6.75it/s] -BigCodeBench-Instruct-calibrated -Groundtruth pass rate: 1.000 -pass@1: 0.568 -``` - -- The "k" includes `[1, 5, 10]` where k values `<=` the sample size will be used -- A cache file named like `samples_eval_results.json` will be cached. Remove it to re-run the evaluation - -
🤔 How long it would take? :: click to expand :: -
- -If you do greedy decoding where there is only one sample for each task, the evaluation should take just a few minutes on Intel(R) Xeon(R) Gold 6150 CPU @ 2.70GHz, composed of 2 sockets, with 18 cores per socket. However, if you have multiple samples for each task, the evaluation will take longer. -Here are some tips to speed up the evaluation: - -* Use `--parallel $(nproc)` -* Use our pre-evaluated results (see [LLM-generated code](#-LLM-generated-code)) - -
-
- -## 🔍 Failure Inspection - -You can inspect the failed samples by using the following command: - -```bash -# Inspect the failed samples and save the results to `inspect/` -bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard - -# Re-run the inspection in place -bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard --in_place -``` - -## 🚀 Full Script - -We provide a sample script to run the full pipeline: - -```bash -bash run.sh -``` - -## 📊 Result Analysis - -We provide a script to replicate the analysis like Elo Rating and Task Solve Rate, which helps you understand the performance of the models further. - -```bash -To run the analysis, you need to put all the `samples_eval_results.json` files in a `results` folder, which is in the same directory as the script. - -```bash -cd analysis -python get_results.py +export GOOGLE_API_KEY= ``` ## 💻 LLM-generated Code @@ -358,15 +144,9 @@ python get_results.py We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): * See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. -## 🐞 Known Issues - -- [ ] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizers may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. - -- [ ] Due to the flakiness in the evaluation, the execution results may vary slightly (~0.2% for Full set, and ~0.6% for Hard set) between runs. We are working on improving the evaluation stability. - -- [x] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. If the issue persists ,please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). +## Advanced Usage -- [x] ~~We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code.~~ Please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). +Please refer to the [ADVANCED USAGE](https://github.com/bigcode-project/bigcodebench/blob/main/ADVANCED_USAGE.md) for more details. ## 📜 Citation diff --git a/_README.md b/_README.md new file mode 100755 index 0000000..c0e80c0 --- /dev/null +++ b/_README.md @@ -0,0 +1,393 @@ +# BigCodeBench +
+BigCodeBench +
+ +

+ + + + + + + + + +

+ +

+ 🌸About • + 🔥Quick Start • + 🔍Failure Inspection • + 🚀Full Script • + 📊Result Analysis • + 💻LLM-generated Code • + 🐞Known Issues • + 📜Citation • + 🙏Acknowledgement +

+ +## 📰 News +- **[2024-10-05]** We create a public code execution API on the Hugging Face space. +- **[2024-10-01]** We have evaluated 139 models on BigCodeBench-Hard so far. Take a look at the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard)! +- **[2024-08-19]** To make the evaluation fully reproducible, we add a real-time code execution session to the leaderboard. It can be viewed [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). +- **[2024-08-02]** We release `bigcodebench==v0.1.9`. + +
More News :: click to expand :: +
+ +- **[2024-07-18]** We announce a subset of BigCodeBench, BigCodeBench-Hard, which includes 148 tasks that are more aligned with the real-world programming tasks. The details are available [in this blog post](https://huggingface.co/blog/terryyz/bigcodebench-hard). The dataset is available [here](https://huggingface.co/datasets/bigcode/bigcodebench-hard). The new release is `bigcodebench==v0.1.8`. +- **[2024-06-28]** We release `bigcodebench==v0.1.7`. +- **[2024-06-27]** We release `bigcodebench==v0.1.6`. +- **[2024-06-19]** We start the Hugging Face BigCodeBench Leaderboard! The leaderboard is available [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). +- **[2024-06-18]** We release BigCodeBench, a new benchmark for code generation with 1140 software-engineering-oriented programming tasks. Preprint is available [here](https://arxiv.org/abs/2406.15877). PyPI package is available [here](https://pypi.org/project/bigcodebench/) with the version `0.1.5`. + +
+
+ +## 🌸 About + +### BigCodeBench + +BigCodeBench is an **_easy-to-use_** benchmark for code generation with **_practical_** and **_challenging_** programming tasks. It aims to evaluate the true programming capabilities of large language models (LLMs) in a more realistic setting. The benchmark is designed for HumanEval-like function-level code generation tasks, but with much more complex instructions and diverse function calls. +To facilitate the evaluation of LLMs on BigCodeBench, we provide this Python package `bigcodebench` that includes the dataset, generation scripts, and evaluation scripts. The package is built on top of the [EvalPlus](https://github.com/evalplus/evalplus) framework, which is a flexible and extensible evaluation framework for code generation tasks. + +### Why BigCodeBench? + +BigCodeBench focuses on task automation via code generation with *diverse function calls* and *complex instructions*, with: + +* ✨ **Precise evaluation & ranking**: See [our leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard) for latest LLM rankings before & after rigorous evaluation. +* ✨ **Pre-generated samples**: BigCodeBench accelerates code intelligence research by open-sourcing [LLM-generated samples](#-LLM-generated-code) for various models -- no need to re-run the expensive benchmarks! + +## 🔥 Quick Start + +> [!Tip] +> +> BigCodeBench ❤️ [bigcode-evaluation-harness](https://github.com/bigcode-project/bigcode-evaluation-harness)! +> BigCodeBench will be integrated to bigcode-evaluation-harness, and you can also run it there! + +To get started, please first set up the environment: + +```bash +# Install to use bigcodebench.evaluate +pip install bigcodebench --upgrade +# If you want to use the evaluate locally, you need to install the requirements +pip install -I -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt + +# Install to use bigcodebench.generate +# You are strongly recommended to install the generate dependencies in a separate environment +pip install bigcodebench[generate] --upgrade +``` + +
⏬ Install nightly version :: click to expand :: +
+ +```bash +# Install to use bigcodebench.evaluate +pip install "git+https://github.com/bigcode-project/bigcodebench.git" --upgrade +``` + +
+
+ +
⏬ Using BigCodeBench as a local repo? :: click to expand :: +
+ +```bash +git clone https://github.com/bigcode-project/bigcodebench.git +cd bigcodebench +export PYTHONPATH=$PYTHONPATH:$(pwd) +# Install to use bigcodebench.evaluate +pip install -e . +# Install to use bigcodebench.generate +pip install -e .[generate] +``` + +
+
+ +### Code Generation + +You are suggested to use `flash-attn` for generating code samples. +```bash +pip install -U flash-attn +``` + +To generate code samples from a model, you can use the following command: +> +```bash +# when greedy, there is no need for temperature and n_samples +bigcodebench.generate \ + --model [model_name] \ + --split [complete|instruct] \ + --subset [full|hard] \ + [--greedy] \ + --bs [bs] \ + --temperature [temp] \ + --n_samples [n_samples] \ + --resume \ + --backend [vllm|hf|openai|mistral|anthropic|google] \ + --tp [gpu_number] \ + [--trust_remote_code] \ + [--base_url [base_url]] \ + [--tokenizer_name [tokenizer_name]] +``` +> +The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: +> +```bash +# If you are using GPUs +docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ + --model [model_name] \ + --split [complete|instruct] \ + --subset [full|hard] \ + [--greedy] \ + --bs [bs] \ + --temperature [temp] \ + --n_samples [n_samples] \ + --resume \ + --backend [vllm|hf|openai|mistral|anthropic|google] \ + --tp [gpu_number] + +# ...Or if you are using CPUs +docker run -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ + --model [model_name] \ + --split [complete|instruct] \ + --subset [full|hard] \ + [--greedy] \ + --bs [bs] \ + --temperature [temp] \ + --n_samples [n_samples] \ + --resume \ + --backend [vllm|hf|openai|mistral|anthropic|google] +``` +> +```bash +# If you wish to use gated or private HuggingFace models and datasets +docker run -e HUGGING_FACE_HUB_TOKEN=$token -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments4 + +# Similarly, to use other backends that require authentication +docker run -e OPENAI_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments +docker run -e GOOGLE_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments +docker run -e ANTHROPIC_KEY=$ANTHROPIC_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments +``` +> +Following which, you can run the built container as shown in above. +> +
🤔 Structure of `problem`? :: click to expand :: +
+ +* `task_id` is the identifier string for the task +* `entry_point` is the name of the function +* `complete_prompt` is the prompt for BigCodeBench-Complete +* `instruct_prompt` is the prompt for BigCodeBench-Instruct ++ `canonical_solution` is the ground-truth implementation ++ `test` is the `unittest.TestCase` class + +
+
+ +> [!Note] +> +> **Expected Schema of `[model_name]--bigcodebench-[task]--[backend]-[temp]-[n_samples].jsonl`** +> +> 1. `task_id`: Task ID, which are the keys of `get_bigcodebench()` +> 2. `solution` (optional): Self-contained solution (usually including the prompt) +> * Example: `{"task_id": "BigCodeBench/?", "solution": "def f():\n return 1"}` + +### Code Post-processing + +LLM-generated text may not be compilable code for including natural language lines or incomplete extra code. +We provide a tool namely `bigcodebench.sanitize` to clean up the code: + +```bash +# 💡 If you want to get the calibrated results: +bigcodebench.sanitize --samples samples.jsonl --calibrate +# Sanitized code will be produced to `samples-sanitized-calibrated.jsonl` + +# 💡 Optionally run the sanitization step with multiprocessing to speedup +bigcodebench.sanitize --samples samples.jsonl --calibrate --parallel 8 + +# 💡 If you want to get the original results: +bigcodebench.sanitize --samples samples.jsonl +# Sanitized code will be produced to `samples-sanitized.jsonl` + +# 💡 If you are storing codes in directories: +bigcodebench.sanitize --samples /path/to/vicuna-[??]b_temp_[??] +# Sanitized code will be produced to `/path/to/vicuna-[??]b_temp_[??]-sanitized` +``` + +If you want to use the pre-built docker images for post-processing, you can use the following command: + +```bash +# Change the entrypoint to bigcodebench.sanitize in any pre-built docker image, like bigcodebench/bigcodebench-evaluate:latest +docker run -it --entrypoint bigcodebench.sanitize -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --samples samples.jsonl +``` + +
🔎 Checking the compatibility of post-processed code:: click to expand :: +
+ +To double-check the post-processing results, you can use `bigcodebench.syncheck` to check the code validity before and after sanitization, which will print erroneous code snippets and why they are wrong: + +```bash +# 💡 If you are storing codes in jsonl: +bigcodebench.syncheck --samples samples.jsonl + +# 💡 If you are storing codes in directories: +bigcodebench.syncheck --samples /path/to/vicuna-[??]b_temp_[??] + +# 💡 Or change the entrypoint to bigcodebench.syncheck in any pre-built docker image, like +docker run -it --entrypoint bigcodebench.syncheck -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --samples samples.jsonl +``` + +
+
+ + +### Code Evaluation + +You are strongly recommended to use a sandbox such as [docker](https://docs.docker.com/get-docker/): + +```bash +# Mount the current directory to the container +# If you want to change the RAM address space limit (in MB, 30 GB by default): `--max-as-limit XXX` +# If you want to change the RAM data segment limit (in MB, 30 GB by default): `--max-data-limit` +# If you want to change the RAM stack limit (in MB, 10 MB by default): `--max-stack-limit` +# If you want to increase the execution time limit (in seconds, 240 seconds by default): `--min-time-limit` +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl + +# If you only want to check the ground truths +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --check-gt-only +``` + +...Or if you want to try it locally regardless of the risks ⚠️: + +First, install the dependencies for BigCodeBench: + +```bash +pip install -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt +``` + +Then, run the evaluation: + +```bash +# ...Or locally ⚠️ +bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl +# ...If you really don't want to check the ground truths +bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt +# If you want to save the pass rate to a file +bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --save_pass_rate + +# You are strongly recommended to use the following command to clean up the environment after evaluation: +pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; +rm -rf /tmp/* +``` + +> [!Tip] +> +> Do you use a very slow machine? +> +> LLM solutions are regarded as **failed** on timeout (and OOM etc.). +> Specifically, we set the dynamic timeout based on the ground-truth solution's runtime. +> +> Additionally, you are **NOT** encouraged to make your test-bed over stressed while running evaluation. +> For example, using `--parallel 64` on a 4-core machine or doing something else during evaluation are bad ideas... + +
⌨️ More command-line flags :: click to expand :: +
+ +* `--parallel`: by default half of the cores + +
+
+ +The output should be like (below is GPT-4 greedy decoding example): + +``` +Asserting the groundtruth... +Expected outputs computed in 1200.0 seconds +Reading samples... +1140it [00:00, 1901.64it/s] +Evaluating samples... +100%|██████████████████████████████████████████| 1140/1140 [19:53<00:00, 6.75it/s] +BigCodeBench-Instruct-calibrated +Groundtruth pass rate: 1.000 +pass@1: 0.568 +``` + +- The "k" includes `[1, 5, 10]` where k values `<=` the sample size will be used +- A cache file named like `samples_eval_results.json` will be cached. Remove it to re-run the evaluation + +
🤔 How long it would take? :: click to expand :: +
+ +If you do greedy decoding where there is only one sample for each task, the evaluation should take just a few minutes on Intel(R) Xeon(R) Gold 6150 CPU @ 2.70GHz, composed of 2 sockets, with 18 cores per socket. However, if you have multiple samples for each task, the evaluation will take longer. +Here are some tips to speed up the evaluation: + +* Use `--parallel $(nproc)` +* Use our pre-evaluated results (see [LLM-generated code](#-LLM-generated-code)) + +
+
+ +## 🔍 Failure Inspection + +You can inspect the failed samples by using the following command: + +```bash +# Inspect the failed samples and save the results to `inspect/` +bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard + +# Re-run the inspection in place +bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard --in_place +``` + +## 🚀 Full Script + +We provide a sample script to run the full pipeline: + +```bash +bash run.sh +``` + +## 📊 Result Analysis + +We provide a script to replicate the analysis like Elo Rating and Task Solve Rate, which helps you understand the performance of the models further. + +```bash +To run the analysis, you need to put all the `samples_eval_results.json` files in a `results` folder, which is in the same directory as the script. + +```bash +cd analysis +python get_results.py +``` + +## 💻 LLM-generated Code + +We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): +* See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. + +## 🐞 Known Issues + +- [x] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizers may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. + +- [x] Due to the flakiness in the evaluation, the execution results may vary slightly (~0.2% for Full set, and ~0.6% for Hard set) between runs. We are working on improving the evaluation stability. + +- [x] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. If the issue persists ,please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). + +- [x] We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code. + +## 📜 Citation + +```bibtex +@article{zhuo2024bigcodebench, + title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}, + author={Zhuo, Terry Yue and Vu, Minh Chien and Chim, Jenny and Hu, Han and Yu, Wenhao and Widyasari, Ratnadira and Yusuf, Imam Nur Bani and Zhan, Haolan and He, Junda and Paul, Indraneil and others}, + journal={arXiv preprint arXiv:2406.15877}, + year={2024} +} +``` + +## 🙏 Acknowledgement + +- [EvalPlus](https://github.com/evalplus/evalplus) From 01e2fb08e33f56198ddb0de862ed9ad43a98cab2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 06:03:21 +0800 Subject: [PATCH 154/325] doc: update full script --- run.sh | 35 +++++------------------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/run.sh b/run.sh index f33fe01..055fd8d 100755 --- a/run.sh +++ b/run.sh @@ -1,39 +1,14 @@ -BS=5 DATASET=bigcodebench -MODEL=gpt-3.5-turbo-0125 -BACKEND=openai -TEMP=0 -N_SAMPLES=1 +MODEL=meta-llama/Llama-3.2-1B-Instruct +BACKEND=vllm NUM_GPU=1 SPLIT=complete SUBSET=hard -if [[ $MODEL == *"/"* ]]; then - ORG=$(echo $MODEL | cut -d'/' -f1)-- - BASE_MODEL=$(echo $MODEL | cut -d'/' -f2) -else - ORG="" - BASE_MODEL=$MODEL -fi -if [ "$SUBSET" = "full" ]; then - FILE_HEADER="${ORG}${BASE_MODEL}--${DATASET}-${SPLIT}--${BACKEND}-${TEMP}-${N_SAMPLES}" - else - FILE_HEADER="${ORG}${BASE_MODEL}--${DATASET}-${SUBSET}-${SPLIT}--${BACKEND}-${TEMP}-${N_SAMPLES}" - fi - -echo $FILE_HEADER -bigcodebench.generate \ +bigcodebench.evaluate \ --model $MODEL \ - --resume \ + --samples meta-llama--Llama-3.2-1B-Instruct--bigcodebench-hard-complete--vllm-0-1-sanitized_calibrated.jsonl \ --split $SPLIT \ --subset $SUBSET \ --backend $BACKEND \ - --greedy - -bigcodebench.sanitize --samples $FILE_HEADER.jsonl --calibrate - -# Check if the ground truth works on your machine -bigcodebench.evaluate --split $SPLIT --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl - -# If the execution is slow: -bigcodebench.evaluate --split $SPLIT --subset $SUBSET --samples $FILE_HEADER-sanitized-calibrated.jsonl --parallel 32 \ No newline at end of file + --greedy \ No newline at end of file From b8f5aea91c85f509e3b50fdff3f64777334fab43 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 06:04:38 +0800 Subject: [PATCH 155/325] doc: minor fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f72ea2a..7cf9eb7 100755 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ bigcodebench.evaluate \ > [!Note] > -> Remotely executing on BigCodeBench-Full typically takes 5-7 minutes, and on BigCodeBench-Hard typically takes 3-5 minutes. +> Remotely executing on `BigCodeBench-Full` typically takes 5-7 minutes, and on `BigCodeBench-Hard` typically takes 3-5 minutes. > [!Note] > From 33548bbd8ca7226b688144b9af665dd658718676 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 06:06:11 +0800 Subject: [PATCH 156/325] doc: minor fix --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 7cf9eb7..33234d8 100755 --- a/README.md +++ b/README.md @@ -46,8 +46,7 @@ ### BigCodeBench -BigCodeBench is an **_easy-to-use_** benchmark for code generation with **_practical_** and **_challenging_** programming tasks. It aims to evaluate the true programming capabilities of large language models (LLMs) in a more realistic setting. The benchmark is designed for HumanEval-like function-level code generation tasks, but with much more complex instructions and diverse function calls. -To facilitate the evaluation of LLMs on BigCodeBench, we provide this Python package `bigcodebench` that includes the dataset, generation scripts, and evaluation scripts. The package is built on top of the [EvalPlus](https://github.com/evalplus/evalplus) framework, which is a flexible and extensible evaluation framework for code generation tasks. +BigCodeBench is an **_easy-to-use_** benchmark for solving **_practical_** and **_challenging_** tasks via code. It aims to evaluate the true programming capabilities of large language models (LLMs) in a more realistic setting. The benchmark is designed for HumanEval-like function-level code generation tasks, but with much more complex instructions and diverse function calls. ### Why BigCodeBench? From 62f8ab1a0079661ad1e58cb411d6a3eff1eaa6b0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:48:44 +0000 Subject: [PATCH 157/325] refactor(tools): rrename to 0.2.0 --- tools/{fix_v0110.py => fix_v020.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/{fix_v0110.py => fix_v020.py} (100%) diff --git a/tools/fix_v0110.py b/tools/fix_v020.py similarity index 100% rename from tools/fix_v0110.py rename to tools/fix_v020.py From 625752eb5a0fd6aa8a75dd71f06d7b1bc86f0637 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:49:56 +0000 Subject: [PATCH 158/325] refactor(gen): remove mistral errors --- bigcodebench/gen/util/mistral_request.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bigcodebench/gen/util/mistral_request.py b/bigcodebench/gen/util/mistral_request.py index e61fce7..a7ea094 100644 --- a/bigcodebench/gen/util/mistral_request.py +++ b/bigcodebench/gen/util/mistral_request.py @@ -1,3 +1,5 @@ +import time + from mistralai.client import MistralClient from mistralai.models.chat_completion import ChatMessage @@ -7,6 +9,7 @@ def make_auto_request(client: MistralClient, *args, **kwargs) -> ChatMessage: try: ret = client.chat(*args, **kwargs) except Exception as e: + print("Unknown error. Waiting...") print(e) time.sleep(1) return ret \ No newline at end of file From d367f5b5bd5ded11eef6263cc506e3de4c7bfa54 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:50:52 +0000 Subject: [PATCH 159/325] update dependencies --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 176759b..a9b7c74 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ install_requires = tree-sitter==0.21.3 wget>=3.2 datasets + gradio-client [options.extras_require] generate = @@ -37,7 +38,7 @@ generate = accelerate>=0.30.1 anthropic>=0.26.1 google-generativeai>=0.5.4 - mistralai>=0.2.0 + mistralai>=0.2.0,<1.0.0 openai>=1.11.1 [options.entry_points] From ed9886cca585d4f6594b989c508fd164d2a6d408 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:51:29 +0000 Subject: [PATCH 160/325] doc: add args details --- ADVANCED_USAGE.md | 68 ++++++++++++++++++++++++++++++++++++----------- README.md | 2 ++ 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 526031b..0f14173 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -43,7 +43,54 @@ pip install -e .[generate]
-### 🚀 Local Generation +## 🚀 Remote Evaluation + +Below are all the arguments for `bigcodebench.evaluate` for the remote evaluation: + +#### Required Arguments: +- `--model`: The model to evaluate +- `--split`: The split of the dataset to evaluate +- `--subset`: The subset of the dataset to evaluate + +#### Optional Arguments: +- `--root`: The root directory to store the results, default to `bcb_results` +- `--bs`: The batch size, default to `1` +- `--n_samples`: The number of samples, default to `1` +- `--temperature`: The temperature, default to `0.0` +- `--max_new_tokens`: The length of max new tokens, default to `1280` +- `--greedy`: Whether to use greedy decoding, default to `False` +- `--strip_newlines`: Whether to strip newlines, default to `False`, set to `True` to strip newlines for some model series like StarCoder2 +- `--direct_completion`: Whether to use direct completion, default to `False` +- `--resume`: Whether to resume the evaluation, default to `True`, set to `False` to re-run the evaluation +- `--id_range`: The range of the tasks to evaluate, default to `None`, e.g. `--id_range 10,20` will evaluate the tasks from 10 to 20 +- `--backend`: The backend to use, default to `vllm` +- `--base_url`: The base URL of the backend for OpenAI-compatible APIs, default to `None` +- `--tp`: The tensor parallel size for the vLLM backend, default to `1` +- `--trust_remote_code`: Whether to trust the remote code, default to `False` +- `--tokenizer_name`: The name of the customized tokenizer, default to `None` +- `--tokenizer_legacy`: Whether to use the legacy tokenizer, default to `False` +- `--samples`: The path to the generated samples file, default to `None` +- `--local_execute`: Whether to execute the samples locally, default to `False` +- `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://github.com/bigcode-project/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page. +- `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` +- `--save_pass_rate`: Whether to save the pass rate to a file, default to `True` +- `--parallel`: The number of parallel processes, default to `None`, e.g. `--parallel 10` will evaluate 10 samples in parallel +- `--min_time_limit`: The minimum time limit for the execution, default to `1`, e.g. `--min_time_limit 10` will evaluate the samples with at least 10 seconds +- `--max_as_limit`: The maximum address space limit for the execution, default to `30*1024` (30 GB), e.g. `--max_as_limit 20*1024` will evaluate the samples with at most 20 GB +- `--max_data_limit`: The maximum data segment limit for the execution, default to `30*1024` (30 GB), e.g. `--max_data_limit 20*1024` will evaluate the samples with at most 20 GB +- `--max_stack_limit`: The maximum stack limit for the execution, default to `10`, e.g. `--max_stack_limit 20` will evaluate the samples with at most 20 MB +- `--check_gt_only`: Whether to only check the ground truths, default to `False` +- `--no_gt`: Whether to not check the ground truths, default to `False` + +## 🚀 Full Script + +We provide an example script to run the full pipeline for the remote evaluation: + +```bash +bash run.sh +``` + +## 🚀 Local Generation ```bash # when greedy, there is no need for temperature and n_samples @@ -62,9 +109,11 @@ bigcodebench.generate \ [--base_url [base_url]] \ [--tokenizer_name [tokenizer_name]] ``` + > The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: > + ```bash # If you are using GPUs docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ @@ -148,7 +197,7 @@ docker run -it --entrypoint bigcodebench.syncheck -v $(pwd):/app bigcodebench/bi -### Local Evaluation +## 🚀 Local Evaluation You are strongly recommended to use a sandbox such as [docker](https://docs.docker.com/get-docker/): @@ -250,14 +299,6 @@ bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.jso bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard --in_place ``` -## 🚀 Full Script - -We provide a sample script to run the full pipeline: - -```bash -bash run.sh -``` - ## 📊 Result Analysis We provide a script to replicate the analysis like Elo Rating and Task Solve Rate, which helps you understand the performance of the models further. @@ -270,12 +311,7 @@ cd analysis python get_results.py ``` -## 💻 LLM-generated Code - -We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): -* See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. - -## 🐞 Known Issues +## 🐞 Resolved Issues - [x] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizers may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. diff --git a/README.md b/README.md index 33234d8..5f361ee 100755 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ bigcodebench.evaluate \ --tp [TENSOR_PARALLEL_SIZE] \ --greedy ``` + +- All the resulted files will be stored in a folder named `bcb_results`. - The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated.jsonl`. - The evaluation results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_eval_results.json`. - The pass@k results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_pass_at_k.json`. From 71097102a6e01baf59cbeccaf90ea20900a827f8 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:53:00 +0000 Subject: [PATCH 161/325] doc: update full script --- run.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/run.sh b/run.sh index 055fd8d..a84199e 100755 --- a/run.sh +++ b/run.sh @@ -1,14 +1,13 @@ DATASET=bigcodebench -MODEL=meta-llama/Llama-3.2-1B-Instruct +MODEL=meta-llama/Meta-Llama-3.1-8B-Instruct BACKEND=vllm -NUM_GPU=1 +NUM_GPU=2 SPLIT=complete SUBSET=hard bigcodebench.evaluate \ --model $MODEL \ - --samples meta-llama--Llama-3.2-1B-Instruct--bigcodebench-hard-complete--vllm-0-1-sanitized_calibrated.jsonl \ --split $SPLIT \ --subset $SUBSET \ --backend $BACKEND \ - --greedy \ No newline at end of file + --tp $NUM_GPU \ No newline at end of file From e51bd31ed70b4b74944c322eaedbdfb0e278bd6d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:57:58 +0000 Subject: [PATCH 162/325] doc: minor update --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5f361ee..d5915d3 100755 --- a/README.md +++ b/README.md @@ -92,13 +92,12 @@ pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcode We use the greedy decoding as an example to show how to evaluate the generated code samples via remote API. ```bash +# greedy decoding by default bigcodebench.evaluate \ --model meta-llama/Meta-Llama-3.1-8B-Instruct \ --split [complete|instruct] \ --subset [full|hard] \ - --backend [vllm|openai|anthropic|google|mistral|hf] \ - --tp [TENSOR_PARALLEL_SIZE] \ - --greedy + --backend [vllm|openai|anthropic|google|mistral|hf] ``` - All the resulted files will be stored in a folder named `bcb_results`. From 7c5c3d08e200743d9ceca4021ebfac5fe8c44fcf Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:58:50 +0000 Subject: [PATCH 163/325] refactor(gen): update model provider --- bigcodebench/provider/__init__.py | 27 ++++++++++------ bigcodebench/provider/anthropic.py | 12 +++---- bigcodebench/provider/base.py | 8 +++-- bigcodebench/provider/google.py | 18 +++++------ bigcodebench/provider/hf.py | 50 ++++++++++++++++++------------ bigcodebench/provider/mistral.py | 16 +++++----- bigcodebench/provider/openai.py | 8 +++-- bigcodebench/provider/utility.py | 17 +++++----- bigcodebench/provider/vllm.py | 1 + 9 files changed, 91 insertions(+), 66 deletions(-) diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index 2203196..67123f9 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -8,19 +8,20 @@ def make_model( split: str, dataset: str = "bigcodebench", temperature: float = 0.0, + max_new_tokens: int = 1280, # instruction model only - instruction_prefix=None, - response_prefix=None, + instruction_prefix: str = None, + response_prefix: str = None, # vllm only - tp=1, - direct_completion=False, - base_url=None, - trust_remote_code=False, + tp: int = 1, + direct_completion: bool = False, + base_url: str = None, + trust_remote_code: bool = False, # hf only - attn_implementation="eager", + attn_implementation: str = "eager", # tokenizer - tokenizer_name=None, - tokenizer_kwargs=None, + tokenizer_name: str = None, + tokenizer_legacy: bool = True, ) -> DecoderBase: if backend == "vllm": from bigcodebench.provider.vllm import VllmDecoder @@ -30,9 +31,10 @@ def make_model( subset=subset, split=split, temperature=temperature, + max_new_tokens=max_new_tokens, dataset=dataset, direct_completion=direct_completion, - tensor_parallel_size=tp, + tp=tp, instruction_prefix=instruction_prefix, response_prefix=response_prefix, ) @@ -44,6 +46,7 @@ def make_model( subset=subset, split=split, temperature=temperature, + max_new_tokens=max_new_tokens, dataset=dataset, direct_completion=direct_completion, instruction_prefix=instruction_prefix, @@ -59,6 +62,7 @@ def make_model( subset=subset, split=split, temperature=temperature, + max_new_tokens=max_new_tokens, base_url=base_url, instruction_prefix=instruction_prefix, response_prefix=response_prefix, @@ -71,6 +75,7 @@ def make_model( subset=subset, split=split, temperature=temperature, + max_new_tokens=max_new_tokens, instruction_prefix=instruction_prefix, response_prefix=response_prefix, ) @@ -83,6 +88,7 @@ def make_model( subset=subset, split=split, temperature=temperature, + max_new_tokens=max_new_tokens, instruction_prefix=instruction_prefix, response_prefix=response_prefix, ) @@ -95,6 +101,7 @@ def make_model( subset=subset, split=split, temperature=temperature, + max_new_tokens=max_new_tokens, instruction_prefix=instruction_prefix, response_prefix=response_prefix, ) \ No newline at end of file diff --git a/bigcodebench/provider/anthropic.py b/bigcodebench/provider/anthropic.py index 3f51ecf..1969e0c 100644 --- a/bigcodebench/provider/anthropic.py +++ b/bigcodebench/provider/anthropic.py @@ -1,9 +1,10 @@ import os from typing import List +from tqdm import tqdm import anthropic -from bigcodebench.gen.util import anthropic_request +from bigcodebench.gen.util.anthropic_request import make_auto_request from bigcodebench.provider.base import DecoderBase from bigcodebench.provider.utility import make_raw_chat_prompt @@ -18,15 +19,12 @@ def codegen( if do_sample: assert self.temperature > 0, "Temperature must be positive for sampling" - if not do_sample: - assert batch_size == 1, "Sampling only supports batch size of 1" - all_outputs = [] for prompt in tqdm(prompts): outputs = [] for _ in range(num_samples): - message = anthropic_request.make_auto_request( + ret = make_auto_request( client=self.client, model=self.name, messages=[ @@ -46,9 +44,9 @@ def codegen( temperature=self.temperature, stop_sequences=self.eos, ) - outputs.append(message.content[0].text) + outputs.append(ret.content[0].text) all_outputs.append(outputs) - return outputs + return all_outputs def is_direct_completion(self) -> bool: return False \ No newline at end of file diff --git a/bigcodebench/provider/base.py b/bigcodebench/provider/base.py index e5bad6d..ebec843 100644 --- a/bigcodebench/provider/base.py +++ b/bigcodebench/provider/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List -from evalplus.provider.utility import EOS +from bigcodebench.provider.utility import EOS class DecoderBase(ABC): @@ -11,12 +11,14 @@ def __init__( subset: str, split: str, temperature: float = 0.8, - max_new_tokens: int = 5120, + max_new_tokens: int = 1280, dtype: str = "bfloat16", # default direct_completion: bool = False, trust_remote_code: bool = False, tokenizer_name: str = None, tokenizer_legacy: bool = False, + instruction_prefix: str = None, + response_prefix: str = None, ) -> None: print("Initializing a decoder model: {} ...".format(name)) self.name = name @@ -31,6 +33,8 @@ def __init__( self.trust_remote_code = trust_remote_code self.tokenizer_name = tokenizer_name self.tokenizer_legacy = tokenizer_legacy + self.instruction_prefix = instruction_prefix + self.response_prefix = response_prefix @abstractmethod def codegen( diff --git a/bigcodebench/provider/google.py b/bigcodebench/provider/google.py index 07237c1..0cd5416 100644 --- a/bigcodebench/provider/google.py +++ b/bigcodebench/provider/google.py @@ -1,9 +1,9 @@ import os from typing import List +from tqdm import tqdm import google.generativeai as genai - from bigcodebench.provider.base import DecoderBase from bigcodebench.gen.util.google_request import make_auto_request from bigcodebench.provider.utility import make_raw_chat_prompt @@ -24,7 +24,7 @@ def codegen( all_outputs = [] for prompt in tqdm(prompts): - ret_texts = [] + outputs = [] message = make_raw_chat_prompt( task_prompt=prompt, subset=self.subset, @@ -33,25 +33,23 @@ def codegen( response_prefix=self.response_prefix, tokenizer=None, ) - replies = make_auto_request( + ret = make_auto_request( self.client, message, self.name, - n=batch_size, + n=num_samples, max_tokens=self.max_new_tokens, temperature=self.temperature, ) - for candidate in replies.candidates: + for candidate in ret.candidates: parts = candidate.content.parts if parts: - ret_texts.append(parts[0].text) + outputs.append(parts[0].text) else: print("Empty response!") - ret_texts.append("") + outputs.append("") print(f"{candidate.safety_ratings = }") - ret_texts.append("") - all_outputs.append(ret_texts + [""] * (batch_size - len(ret_texts))) - + all_outputs.append(outputs) return all_outputs def is_direct_completion(self) -> bool: diff --git a/bigcodebench/provider/hf.py b/bigcodebench/provider/hf.py index b260360..c3136c8 100644 --- a/bigcodebench/provider/hf.py +++ b/bigcodebench/provider/hf.py @@ -4,8 +4,8 @@ from stop_sequencer import StopSequencer from transformers import AutoModelForCausalLM, AutoTokenizer -from evalplus.provider.base import DecoderBase -from evalplus.provider.utility import ( +from bigcodebench.provider.base import DecoderBase +from bigcodebench.provider.utility import ( extra_eos_for_direct_completion, make_raw_chat_prompt, ) @@ -33,6 +33,10 @@ def __init__( print(f"{kwargs = }") self.tokenizer = AutoTokenizer.from_pretrained(name, use_fast=False, legacy=self.tokenizer_legacy) + self.tokenizer.pad_token = self.tokenizer.eos_token + # assume the model is decoder-only + self.tokenizer.padding_side = 'left' + if self.is_direct_completion(): # no chat template self.eos += extra_eos_for_direct_completion(dataset) else: # with chat template @@ -40,7 +44,6 @@ def __init__( print(f"{self.eos = }") self.model = AutoModelForCausalLM.from_pretrained(name, **kwargs) - self.model = self.model.to(self.device) def is_direct_completion(self) -> bool: return self.direct_completion or self.tokenizer.chat_template is None @@ -61,15 +64,16 @@ def codegen( ) for prompt in prompts ] - input_tokens = self.tokenizer.encode(prompts, return_tensors="pt").to( + + input_tokens = self.tokenizer(prompts, return_tensors="pt", padding=True, truncation=True).to( self.device - ) + )["input_ids"] + kwargs = {} if do_sample: kwargs["top_p"] = 0.95 kwargs["temperature"] = self.temperature - - outputs = self.model.generate( + ret = self.model.generate( input_tokens, max_new_tokens=self.max_new_tokens, do_sample=do_sample, @@ -79,17 +83,23 @@ def codegen( tokenizer=self.tokenizer, **kwargs, ) + + # Reshape ret into a list of lists, each sublist containing num_samples elements + ret_chunks = [ret[i:i + num_samples] for i in range(0, len(ret), num_samples)] - gen_strs = self.tokenizer.batch_decode( - outputs[:, input_tokens.size(-1) :], - skip_special_tokens=self.skip_special_tokens, - ) - outputs = [] - # removes eos tokens. - for output in gen_strs: - min_index = 10000 - for eos in self.eos: - if eos in output: - min_index = min(min_index, output.index(eos)) - outputs.append(output[:min_index].replace("\t", " ")) - return outputs \ No newline at end of file + all_outputs = [] + # Process each chunk in ret_chunks + for i, ret_chunk in enumerate(ret_chunks): + gen_strs = self.tokenizer.batch_decode( + ret_chunk[:, input_tokens[i].size(-1):], + skip_special_tokens=self.skip_special_tokens, + ) + outputs = [] + for output in gen_strs: + min_index = 10000 + for eos in self.eos: + if eos in output: + min_index = min(min_index, output.index(eos)) + outputs.append(output[:min_index].replace("\t", " ")) + all_outputs.append(outputs) + return all_outputs \ No newline at end of file diff --git a/bigcodebench/provider/mistral.py b/bigcodebench/provider/mistral.py index c016e2e..9499429 100644 --- a/bigcodebench/provider/mistral.py +++ b/bigcodebench/provider/mistral.py @@ -1,18 +1,21 @@ import os from typing import List +from tqdm import tqdm -import anthropic +from mistralai.client import MistralClient +from mistralai.models.chat_completion import ChatMessage from bigcodebench.provider.base import DecoderBase +from bigcodebench.gen.util.mistral_request import make_auto_request from bigcodebench.provider.utility import make_raw_chat_prompt -class MistralDecoder(DecoderBase): +class MistralChatDecoder(DecoderBase): def __init__(self, name: str, **kwargs) -> None: super().__init__(name, **kwargs) - self.client = mistral.Mistral(api_key=os.getenv("MISTRAL_API_KEY")) + self.client = MistralClient(api_key=os.getenv("MISTRAL_API_KEY")) def codegen( - self, prompt: str, do_sample: bool = True, num_samples: int = 200 + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: if do_sample: assert self.temperature > 0, "Temperature must be positive for sampling" @@ -22,7 +25,7 @@ def codegen( outputs = [] for _ in range(num_samples): - message = mistral_request.make_auto_request( + ret = make_auto_request( client=self.client, model=self.name, messages=[ @@ -40,9 +43,8 @@ def codegen( ) ], max_tokens=self.max_new_tokens, - **kwargs, ) - outputs.append(message.content[0].text) + outputs.append(ret.choices[0].message.content) all_outputs.append(outputs) return all_outputs diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 3f55659..9eba02e 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -1,10 +1,11 @@ import os from typing import List +from tqdm import tqdm import openai -from evalplus.gen.util import openai_request -from evalplus.provider.base import DecoderBase +from bigcodebench.provider.base import DecoderBase +from bigcodebench.gen.util.openai_request import make_auto_request from bigcodebench.provider.utility import make_raw_chat_prompt class OpenAIChatDecoder(DecoderBase): @@ -21,6 +22,7 @@ def codegen( assert self.temperature > 0, "Temperature must be positive for sampling" all_outputs = [] for prompt in tqdm(prompts): + outputs = [] message = make_raw_chat_prompt( task_prompt=prompt, subset=self.subset, @@ -29,7 +31,7 @@ def codegen( response_prefix=self.response_prefix, tokenizer=None, ) - ret = openai_request.make_auto_request( + ret = make_auto_request( self.client, message=message, model=self.name, diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py index 93a739b..2a7aa1c 100644 --- a/bigcodebench/provider/utility.py +++ b/bigcodebench/provider/utility.py @@ -1,4 +1,5 @@ from typing import List +from transformers import AutoTokenizer EOS = [ "<|endoftext|>", @@ -55,11 +56,13 @@ def make_raw_chat_prompt( {_MAGIC_SPLITTER_} ``` """ - task_prompt = tokenizer.apply_chat_template( - [ - {"role": "user", "content": task_prompt}, - {"role": "assistant", "content": response}, - ], - tokenize=False, - ).split(_MAGIC_SPLITTER_)[0] + if tokenizer: + task_prompt = tokenizer.apply_chat_template( + [ + {"role": "user", "content": task_prompt}, + # {"role": "assistant", "content": response}, + ], + tokenize=False, + add_generation_prompt=True + ).split(_MAGIC_SPLITTER_)[0] return task_prompt \ No newline at end of file diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 178b3b6..3d0aaf4 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -1,3 +1,4 @@ +import os from typing import List from transformers import AutoTokenizer From 8993dcaa26f9d8fecc2cac4181d8baacf0769a73 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 15:59:17 +0000 Subject: [PATCH 164/325] refactor: update evaluate pipeline --- bigcodebench/evaluate.py | 5 +++-- bigcodebench/generate.py | 31 ++++++++++++++++++------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 1cee1b4..df8ad85 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -127,7 +127,8 @@ def evaluate( check_gt_only: bool = False, no_gt: bool = False, **model_kwargs, -): +): + if not samples and model_kwargs: samples = run_codegen( split=split, @@ -164,7 +165,7 @@ def evaluate( else: - pass_k = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()] + pass_k = [int(k) for k in pass_k.split(",")] if parallel is None: n_workers = max(1, multiprocessing.cpu_count() // 2) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index a110e90..6333261 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -19,13 +19,13 @@ def codegen( model: DecoderBase, target_path: str, split: str, - subset="full", - greedy=False, - strip_newlines=False, - n_samples=1, - id_range=None, - resume=True, - batch_size: int=-1, + subset: str, + greedy: bool = False, + strip_newlines: bool = False, + n_samples: int = 1, + id_range: Tuple[int, int] = None, + resume: bool = True, + batch_size: int = -1, ): with Progress( TextColumn(f"BigCodeBench--{split.capitalize()} ({subset.capitalize()}) •" + "[progress.percentage]{task.percentage:>3.0f}%"), @@ -51,12 +51,12 @@ def codegen( batch_entry_points = [] # Read existing data once if resuming - existing_data = {} + task2nexist = {} if resume and os.path.exists(target_path): with open(target_path, "r") as f: for line in f: item = json.loads(line) - existing_data[item["task_id"]] = existing_data.get(item["task_id"], 0) + 1 + task2nexist[item["task_id"]] = task2nexist.get(item["task_id"], 0) + 1 for id_num, (task_id, task) in enumerate(p.track(dataset.items())): if id_range is not None: @@ -69,7 +69,7 @@ def codegen( p_name = task_id.replace("/", "_") - n_existing = existing_data.get(task_id, 0) + n_existing = task2nexist.get(task_id, 0) nsamples = n_samples - n_existing try: @@ -91,7 +91,7 @@ def codegen( p.console.print(log) if (batch_size and len(batch_prompts) == batch_size) or id_num == len(dataset) - 1 or (id_range and id_num == id_range[1] - 1): - if not batch_prompts and id_num == len(dataset) - 1: + if not batch_prompts and (id_num == len(dataset) - 1 or (id_range and id_num == id_range[1] - 1)): break outputs = model.codegen( batch_prompts, @@ -130,6 +130,7 @@ def run_codegen( bs: Optional[int] = None, n_samples: int = 1, temperature: float = 0.0, + max_new_tokens: int = 1280, greedy: bool = False, strip_newlines: bool = False, direct_completion: bool = False, @@ -147,7 +148,7 @@ def run_codegen( temperature = 0 n_samples = 1 greedy = True - print("Greedy decoding ON (--greedy): setting bs=1, n_samples=1, temperature=0") + print("Greedy decoding ON (--greedy): setting n_samples=1, temperature=0") if id_range is not None: assert len(id_range) == 2, "id_range must be a list of length 2" @@ -167,6 +168,7 @@ def run_codegen( subset=subset, split=split, temperature=temperature, + max_new_tokens=max_new_tokens, instruction_prefix=instruction_prefix, response_prefix=response_prefix, base_url=base_url, @@ -181,7 +183,10 @@ def run_codegen( identifier = model.replace("/", "--") + f"--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" target_path = os.path.join(root, identifier) - + + if not resume: + os.remove(target_path) + codegen( model=model_runner, target_path=target_path, From 098ee793b2dcd9308421c5ee6b93b02b5264e6dd Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 5 Oct 2024 16:00:02 +0000 Subject: [PATCH 165/325] minor fix --- bigcodebench/provider/utility.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py index 2a7aa1c..60a00e5 100644 --- a/bigcodebench/provider/utility.py +++ b/bigcodebench/provider/utility.py @@ -60,9 +60,8 @@ def make_raw_chat_prompt( task_prompt = tokenizer.apply_chat_template( [ {"role": "user", "content": task_prompt}, - # {"role": "assistant", "content": response}, + {"role": "assistant", "content": response}, ], tokenize=False, - add_generation_prompt=True ).split(_MAGIC_SPLITTER_)[0] return task_prompt \ No newline at end of file From 3d04478148175ced53b0a0bb90491836d603b626 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:02:23 +0800 Subject: [PATCH 166/325] doc: minor update --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d5915d3..fd1c245 100755 --- a/README.md +++ b/README.md @@ -91,8 +91,11 @@ pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcode We use the greedy decoding as an example to show how to evaluate the generated code samples via remote API. +> [!Note] +> +> Remotely executing on `BigCodeBench-Full` typically takes 6-7 minutes, and on `BigCodeBench-Hard` typically takes 4-5 minutes. + ```bash -# greedy decoding by default bigcodebench.evaluate \ --model meta-llama/Meta-Llama-3.1-8B-Instruct \ --split [complete|instruct] \ @@ -105,10 +108,6 @@ bigcodebench.evaluate \ - The evaluation results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_eval_results.json`. - The pass@k results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_pass_at_k.json`. -> [!Note] -> -> Remotely executing on `BigCodeBench-Full` typically takes 5-7 minutes, and on `BigCodeBench-Hard` typically takes 3-5 minutes. - > [!Note] > > BigCodeBench uses different prompts for base and chat models. From 5a5021252b2bb1dafd11f3b658bac6fe1ac7f99c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:02:57 +0800 Subject: [PATCH 167/325] doc: minor update --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index fd1c245..9e82a8e 100755 --- a/README.md +++ b/README.md @@ -57,11 +57,6 @@ BigCodeBench focuses on task automation via code generation with *diverse functi ## 🔥 Quick Start -> [!Tip] -> -> BigCodeBench ❤️ [bigcode-evaluation-harness](https://github.com/bigcode-project/bigcode-evaluation-harness)! -> BigCodeBench will be integrated to bigcode-evaluation-harness, and you can also run it there! - To get started, please first set up the environment: ```bash From 224320394582748c71ec2ac1914d7dd8d566e4c3 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:07:10 +0800 Subject: [PATCH 168/325] doc: minor update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e82a8e..0a5ac62 100755 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@

## 📰 News -- **[2024-10-05]** We create a public code execution API on the Hugging Face space. +- **[2024-10-05]** We create a public code execution API on the [Hugging Face space](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator). - **[2024-10-01]** We have evaluated 139 models on BigCodeBench-Hard so far. Take a look at the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard)! - **[2024-08-19]** To make the evaluation fully reproducible, we add a real-time code execution session to the leaderboard. It can be viewed [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). - **[2024-08-02]** We release `bigcodebench==v0.1.9`. From 549ba7526439998738d88d0749945f3a006571f2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:10:22 +0800 Subject: [PATCH 169/325] rm: old doc --- _README.md | 393 ----------------------------------------------------- 1 file changed, 393 deletions(-) delete mode 100755 _README.md diff --git a/_README.md b/_README.md deleted file mode 100755 index c0e80c0..0000000 --- a/_README.md +++ /dev/null @@ -1,393 +0,0 @@ -# BigCodeBench -
-BigCodeBench -
- -

- - - - - - - - - -

- -

- 🌸About • - 🔥Quick Start • - 🔍Failure Inspection • - 🚀Full Script • - 📊Result Analysis • - 💻LLM-generated Code • - 🐞Known Issues • - 📜Citation • - 🙏Acknowledgement -

- -## 📰 News -- **[2024-10-05]** We create a public code execution API on the Hugging Face space. -- **[2024-10-01]** We have evaluated 139 models on BigCodeBench-Hard so far. Take a look at the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard)! -- **[2024-08-19]** To make the evaluation fully reproducible, we add a real-time code execution session to the leaderboard. It can be viewed [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). -- **[2024-08-02]** We release `bigcodebench==v0.1.9`. - -
More News :: click to expand :: -
- -- **[2024-07-18]** We announce a subset of BigCodeBench, BigCodeBench-Hard, which includes 148 tasks that are more aligned with the real-world programming tasks. The details are available [in this blog post](https://huggingface.co/blog/terryyz/bigcodebench-hard). The dataset is available [here](https://huggingface.co/datasets/bigcode/bigcodebench-hard). The new release is `bigcodebench==v0.1.8`. -- **[2024-06-28]** We release `bigcodebench==v0.1.7`. -- **[2024-06-27]** We release `bigcodebench==v0.1.6`. -- **[2024-06-19]** We start the Hugging Face BigCodeBench Leaderboard! The leaderboard is available [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). -- **[2024-06-18]** We release BigCodeBench, a new benchmark for code generation with 1140 software-engineering-oriented programming tasks. Preprint is available [here](https://arxiv.org/abs/2406.15877). PyPI package is available [here](https://pypi.org/project/bigcodebench/) with the version `0.1.5`. - -
-
- -## 🌸 About - -### BigCodeBench - -BigCodeBench is an **_easy-to-use_** benchmark for code generation with **_practical_** and **_challenging_** programming tasks. It aims to evaluate the true programming capabilities of large language models (LLMs) in a more realistic setting. The benchmark is designed for HumanEval-like function-level code generation tasks, but with much more complex instructions and diverse function calls. -To facilitate the evaluation of LLMs on BigCodeBench, we provide this Python package `bigcodebench` that includes the dataset, generation scripts, and evaluation scripts. The package is built on top of the [EvalPlus](https://github.com/evalplus/evalplus) framework, which is a flexible and extensible evaluation framework for code generation tasks. - -### Why BigCodeBench? - -BigCodeBench focuses on task automation via code generation with *diverse function calls* and *complex instructions*, with: - -* ✨ **Precise evaluation & ranking**: See [our leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard) for latest LLM rankings before & after rigorous evaluation. -* ✨ **Pre-generated samples**: BigCodeBench accelerates code intelligence research by open-sourcing [LLM-generated samples](#-LLM-generated-code) for various models -- no need to re-run the expensive benchmarks! - -## 🔥 Quick Start - -> [!Tip] -> -> BigCodeBench ❤️ [bigcode-evaluation-harness](https://github.com/bigcode-project/bigcode-evaluation-harness)! -> BigCodeBench will be integrated to bigcode-evaluation-harness, and you can also run it there! - -To get started, please first set up the environment: - -```bash -# Install to use bigcodebench.evaluate -pip install bigcodebench --upgrade -# If you want to use the evaluate locally, you need to install the requirements -pip install -I -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt - -# Install to use bigcodebench.generate -# You are strongly recommended to install the generate dependencies in a separate environment -pip install bigcodebench[generate] --upgrade -``` - -
⏬ Install nightly version :: click to expand :: -
- -```bash -# Install to use bigcodebench.evaluate -pip install "git+https://github.com/bigcode-project/bigcodebench.git" --upgrade -``` - -
-
- -
⏬ Using BigCodeBench as a local repo? :: click to expand :: -
- -```bash -git clone https://github.com/bigcode-project/bigcodebench.git -cd bigcodebench -export PYTHONPATH=$PYTHONPATH:$(pwd) -# Install to use bigcodebench.evaluate -pip install -e . -# Install to use bigcodebench.generate -pip install -e .[generate] -``` - -
-
- -### Code Generation - -You are suggested to use `flash-attn` for generating code samples. -```bash -pip install -U flash-attn -``` - -To generate code samples from a model, you can use the following command: -> -```bash -# when greedy, there is no need for temperature and n_samples -bigcodebench.generate \ - --model [model_name] \ - --split [complete|instruct] \ - --subset [full|hard] \ - [--greedy] \ - --bs [bs] \ - --temperature [temp] \ - --n_samples [n_samples] \ - --resume \ - --backend [vllm|hf|openai|mistral|anthropic|google] \ - --tp [gpu_number] \ - [--trust_remote_code] \ - [--base_url [base_url]] \ - [--tokenizer_name [tokenizer_name]] -``` -> -The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: -> -```bash -# If you are using GPUs -docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ - --model [model_name] \ - --split [complete|instruct] \ - --subset [full|hard] \ - [--greedy] \ - --bs [bs] \ - --temperature [temp] \ - --n_samples [n_samples] \ - --resume \ - --backend [vllm|hf|openai|mistral|anthropic|google] \ - --tp [gpu_number] - -# ...Or if you are using CPUs -docker run -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest \ - --model [model_name] \ - --split [complete|instruct] \ - --subset [full|hard] \ - [--greedy] \ - --bs [bs] \ - --temperature [temp] \ - --n_samples [n_samples] \ - --resume \ - --backend [vllm|hf|openai|mistral|anthropic|google] -``` -> -```bash -# If you wish to use gated or private HuggingFace models and datasets -docker run -e HUGGING_FACE_HUB_TOKEN=$token -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments4 - -# Similarly, to use other backends that require authentication -docker run -e OPENAI_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments -docker run -e GOOGLE_API_KEY=$OPENAI_API_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments -docker run -e ANTHROPIC_KEY=$ANTHROPIC_KEY -v $(pwd):/app -t bigcodebench/bigcodebench-generate:latest # omit other arguments -``` -> -Following which, you can run the built container as shown in above. -> -
🤔 Structure of `problem`? :: click to expand :: -
- -* `task_id` is the identifier string for the task -* `entry_point` is the name of the function -* `complete_prompt` is the prompt for BigCodeBench-Complete -* `instruct_prompt` is the prompt for BigCodeBench-Instruct -+ `canonical_solution` is the ground-truth implementation -+ `test` is the `unittest.TestCase` class - -
-
- -> [!Note] -> -> **Expected Schema of `[model_name]--bigcodebench-[task]--[backend]-[temp]-[n_samples].jsonl`** -> -> 1. `task_id`: Task ID, which are the keys of `get_bigcodebench()` -> 2. `solution` (optional): Self-contained solution (usually including the prompt) -> * Example: `{"task_id": "BigCodeBench/?", "solution": "def f():\n return 1"}` - -### Code Post-processing - -LLM-generated text may not be compilable code for including natural language lines or incomplete extra code. -We provide a tool namely `bigcodebench.sanitize` to clean up the code: - -```bash -# 💡 If you want to get the calibrated results: -bigcodebench.sanitize --samples samples.jsonl --calibrate -# Sanitized code will be produced to `samples-sanitized-calibrated.jsonl` - -# 💡 Optionally run the sanitization step with multiprocessing to speedup -bigcodebench.sanitize --samples samples.jsonl --calibrate --parallel 8 - -# 💡 If you want to get the original results: -bigcodebench.sanitize --samples samples.jsonl -# Sanitized code will be produced to `samples-sanitized.jsonl` - -# 💡 If you are storing codes in directories: -bigcodebench.sanitize --samples /path/to/vicuna-[??]b_temp_[??] -# Sanitized code will be produced to `/path/to/vicuna-[??]b_temp_[??]-sanitized` -``` - -If you want to use the pre-built docker images for post-processing, you can use the following command: - -```bash -# Change the entrypoint to bigcodebench.sanitize in any pre-built docker image, like bigcodebench/bigcodebench-evaluate:latest -docker run -it --entrypoint bigcodebench.sanitize -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --samples samples.jsonl -``` - -
🔎 Checking the compatibility of post-processed code:: click to expand :: -
- -To double-check the post-processing results, you can use `bigcodebench.syncheck` to check the code validity before and after sanitization, which will print erroneous code snippets and why they are wrong: - -```bash -# 💡 If you are storing codes in jsonl: -bigcodebench.syncheck --samples samples.jsonl - -# 💡 If you are storing codes in directories: -bigcodebench.syncheck --samples /path/to/vicuna-[??]b_temp_[??] - -# 💡 Or change the entrypoint to bigcodebench.syncheck in any pre-built docker image, like -docker run -it --entrypoint bigcodebench.syncheck -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --samples samples.jsonl -``` - -
-
- - -### Code Evaluation - -You are strongly recommended to use a sandbox such as [docker](https://docs.docker.com/get-docker/): - -```bash -# Mount the current directory to the container -# If you want to change the RAM address space limit (in MB, 30 GB by default): `--max-as-limit XXX` -# If you want to change the RAM data segment limit (in MB, 30 GB by default): `--max-data-limit` -# If you want to change the RAM stack limit (in MB, 10 MB by default): `--max-stack-limit` -# If you want to increase the execution time limit (in seconds, 240 seconds by default): `--min-time-limit` -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl - -# If you only want to check the ground truths -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --check-gt-only -``` - -...Or if you want to try it locally regardless of the risks ⚠️: - -First, install the dependencies for BigCodeBench: - -```bash -pip install -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt -``` - -Then, run the evaluation: - -```bash -# ...Or locally ⚠️ -bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl -# ...If you really don't want to check the ground truths -bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt -# If you want to save the pass rate to a file -bigcodebench.evaluate --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --save_pass_rate - -# You are strongly recommended to use the following command to clean up the environment after evaluation: -pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; -rm -rf /tmp/* -``` - -> [!Tip] -> -> Do you use a very slow machine? -> -> LLM solutions are regarded as **failed** on timeout (and OOM etc.). -> Specifically, we set the dynamic timeout based on the ground-truth solution's runtime. -> -> Additionally, you are **NOT** encouraged to make your test-bed over stressed while running evaluation. -> For example, using `--parallel 64` on a 4-core machine or doing something else during evaluation are bad ideas... - -
⌨️ More command-line flags :: click to expand :: -
- -* `--parallel`: by default half of the cores - -
-
- -The output should be like (below is GPT-4 greedy decoding example): - -``` -Asserting the groundtruth... -Expected outputs computed in 1200.0 seconds -Reading samples... -1140it [00:00, 1901.64it/s] -Evaluating samples... -100%|██████████████████████████████████████████| 1140/1140 [19:53<00:00, 6.75it/s] -BigCodeBench-Instruct-calibrated -Groundtruth pass rate: 1.000 -pass@1: 0.568 -``` - -- The "k" includes `[1, 5, 10]` where k values `<=` the sample size will be used -- A cache file named like `samples_eval_results.json` will be cached. Remove it to re-run the evaluation - -
🤔 How long it would take? :: click to expand :: -
- -If you do greedy decoding where there is only one sample for each task, the evaluation should take just a few minutes on Intel(R) Xeon(R) Gold 6150 CPU @ 2.70GHz, composed of 2 sockets, with 18 cores per socket. However, if you have multiple samples for each task, the evaluation will take longer. -Here are some tips to speed up the evaluation: - -* Use `--parallel $(nproc)` -* Use our pre-evaluated results (see [LLM-generated code](#-LLM-generated-code)) - -
-
- -## 🔍 Failure Inspection - -You can inspect the failed samples by using the following command: - -```bash -# Inspect the failed samples and save the results to `inspect/` -bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard - -# Re-run the inspection in place -bigcodebench.inspect --eval_results sample-sanitized-calibrated_eval_results.json --split complete --subset hard --in_place -``` - -## 🚀 Full Script - -We provide a sample script to run the full pipeline: - -```bash -bash run.sh -``` - -## 📊 Result Analysis - -We provide a script to replicate the analysis like Elo Rating and Task Solve Rate, which helps you understand the performance of the models further. - -```bash -To run the analysis, you need to put all the `samples_eval_results.json` files in a `results` folder, which is in the same directory as the script. - -```bash -cd analysis -python get_results.py -``` - -## 💻 LLM-generated Code - -We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): -* See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. - -## 🐞 Known Issues - -- [x] Due to [the Hugging Face tokenizer update](https://github.com/huggingface/transformers/pull/31305), some tokenizers may be broken and will degrade the performance of the evaluation. Therefore, we set up with `legacy=False` for the initialization. If you notice the unexpected behaviors, please try `--tokenizer_legacy` during the generation. - -- [x] Due to the flakiness in the evaluation, the execution results may vary slightly (~0.2% for Full set, and ~0.6% for Hard set) between runs. We are working on improving the evaluation stability. - -- [x] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. If the issue persists ,please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). - -- [x] We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code. - -## 📜 Citation - -```bibtex -@article{zhuo2024bigcodebench, - title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}, - author={Zhuo, Terry Yue and Vu, Minh Chien and Chim, Jenny and Hu, Han and Yu, Wenhao and Widyasari, Ratnadira and Yusuf, Imam Nur Bani and Zhan, Haolan and He, Junda and Paul, Indraneil and others}, - journal={arXiv preprint arXiv:2406.15877}, - year={2024} -} -``` - -## 🙏 Acknowledgement - -- [EvalPlus](https://github.com/evalplus/evalplus) From c436061f754cf3917c00f5e8a0b65dde36503638 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:12:16 +0800 Subject: [PATCH 170/325] doc: update adv use --- ADVANCED_USAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 0f14173..428cf28 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -319,7 +319,7 @@ python get_results.py - [x] You may get errors like `ImportError: /usr/local/lib/python3.10/site-packages/matplotlib/_c_internal_utils.cpython-310-x86_64-linux-gnu.so: failed to map segment from shared object` when running the evaluation. This is due to the memory limit of the docker container. You can increase the memory limit of the docker container to solve this issue. If the issue persists ,please use the real-time code execution session to evaluate the code in the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). -- [x] We are aware of the issue of some users needing to use a proxy to access the internet. We are working on a subset of the tasks that do not require internet access to evaluate the code. +- [x] We are aware of the issue of some users needing to use a proxy to access the internet. Please use [Remote Evaluation](#-remote-evaluation) to get the accurate results. ## 📜 Citation From 49e3b3caa50a514b712b615c4bc301c3976a5904 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:20:28 +0800 Subject: [PATCH 171/325] rfactor(analysis): update get results --- analysis/get_results.py | 49 +++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index 664e156..e67fa2a 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -11,6 +11,8 @@ import math from datasets import Dataset, DatasetDict, load_dataset from transformers import AutoTokenizer +from cuml.linear_model import LogisticRegression +import cupy as cp def update_model_info(model_info): for model, info in model_info.items(): @@ -67,6 +69,8 @@ def get_results(tids): data = json.load(f) status = [] + if len(data["eval"]) < len(tids): + continue for key, value in data["eval"].items(): if key not in tids: continue @@ -163,23 +167,23 @@ def read_task_perf(tids, task="complete"): try: try: try: - if info["prompted"]:# and not info["direct_complete"]: - files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_hard_eval_results.json") + if info["prompted"]: + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") if files: file = files[0] else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_hard_eval_results.json")[0] + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_hard_eval_results.json")[0] + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] except: - if info["prompted"]: - files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") + if info["prompted"]:# and not info["direct_complete"]: + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_hard_eval_results.json") if files: file = files[0] else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_hard_eval_results.json")[0] else: - file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] + file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_hard_eval_results.json")[0] except: try: if info["prompted"]:# and not info["direct_complete"]: @@ -205,6 +209,9 @@ def read_task_perf(tids, task="complete"): result_files.append(file) with open(file, "r") as f: data = json.load(f) + + if len(data["eval"]) < len(tids): + continue for task_id, perfs in data["eval"].items(): if task_id in tids: status = 1 if perfs[0]["status"] == "pass" else 0 @@ -271,25 +278,26 @@ def get_bootstrap_result(battles, func_compute_elo, num_round): def get_elo_mle(df, SCALE=400, BASE=10, INIT_RATING=1000): - from sklearn.linear_model import LogisticRegression + + models = pd.concat([df["model_a"], df["model_b"]]).unique() models = pd.Series(np.arange(len(models)), index=models) p = len(models.index) n = df.shape[0] - X = np.zeros([n, p]) - X[np.arange(n), models[df["model_a"]]] = +math.log(BASE) - X[np.arange(n), models[df["model_b"]]] = -math.log(BASE) + X = cp.zeros([n, p]) + X[cp.arange(n), models[df["model_a"]]] = +math.log(BASE) + X[cp.arange(n), models[df["model_b"]]] = -math.log(BASE) - Y = np.zeros(n) + Y = cp.zeros(n) Y[df["winner"] == "model_a"] = 1.0 lr = LogisticRegression(fit_intercept=False) - lr.fit(X,Y) + lr.fit(X, Y) elo_scores = SCALE * lr.coef_[0] + INIT_RATING - return pd.Series(elo_scores, index = models.index).sort_values(ascending=False) + return pd.Series(cp.asnumpy(elo_scores), index=models.index).sort_values(ascending=False) def update_elo_rating(results, elo_dict): @@ -387,11 +395,10 @@ def get_perf_df(data_dict): if __name__ == "__main__": - # bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.0_hf") - bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.0_hf") - # model_info = update_model_info(model_info) + bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.1") + bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.1") bcb_config = { - # "": bcb_orig, + "": bcb_orig, "-hard": bcb_hard, } for suffix, bcb in bcb_config.items(): @@ -401,9 +408,9 @@ def get_perf_df(data_dict): instruct_data, instruct_files = read_task_perf(bcb["task_id"], "instruct") complete_df = get_perf_df(complete_data) instruct_df = get_perf_df(instruct_data) + push_ds(DatasetDict({"complete": Dataset.from_pandas(complete_df), "instruct": Dataset.from_pandas(instruct_df)}), f"bigcode/bigcodebench{suffix}-perf") - assert len(model_info) == len(complete_data),\ - f"Missing results for {set([val['name'] for val in model_info.values()]) - set([model for model in complete_data.keys()])}" + with open("task2domain.json", "r") as f: task2domain = json.load(f) domain_complete = get_domain_perf(complete_data, task2domain) From fce1f38d0055ecabf1a838df046cdb5096730eb7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:29:11 +0800 Subject: [PATCH 172/325] update model list --- analysis/utils.py | 108 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index 4bac8d1..ce81bd6 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1169,4 +1169,112 @@ "act_param": None, "open-data": "None", }, + "Qwen/Qwen2.5-Coder-1.5B-Instruct": { + "name": "Qwen2.5-Coder-1.5B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct", + "prompted": True, + "moe": False, + "size": 1.5, + "act_param": 1.5, + "open-data": "None", + }, + "Qwen/Qwen2.5-Coder-7B-Instruct": { + "name": "Qwen2.5-Coder-7B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct", + "prompted": True, + "moe": False, + "size": 7, + "act_param": 7, + "open-data": "None", + }, + "gemini-1.5-pro-002": { + "name": "Gemini-1.5-Pro-002", + "link": "https://deepmind.google/technologies/gemini/pro", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "mistralai/Mistral-Small-Instruct-2409": { + "name": "Mistral-Small-Instruct-2409", + "link": "https://huggingface.co/mistralai/Mistral-Small-Instruct-2409", + "prompted": True, + "moe": False, + "size": 22.2, + "act_param": 22.2, + "open-data": "None", + }, + "Qwen/Qwen2.5-0.5B-Instruct": { + "name": "Qwen2.5-0.5B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct", + "prompted": True, + "moe": False, + "size": 0.5, + "act_param": 0.5, + "open-data": "None", + }, + "Qwen/Qwen2.5-1.5B-Instruct": { + "name": "Qwen2.5-1.5B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct", + "prompted": True, + "moe": False, + "size": 1.5, + "act_param": 1.5, + "open-data": "None", + }, + "Qwen/Qwen2.5-7B-Instruct": { + "name": "Qwen2.5-7B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", + "prompted": True, + "moe": False, + "size": 7, + "act_param": 7, + "open-data": "None", + }, + "Qwen/Qwen2.5-14B-Instruct": { + "name": "Qwen2.5-14B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct", + "prompted": True, + "moe": False, + "size": 14, + "act_param": 14, + "open-data": "None", + }, + "Qwen/Qwen2.5-32B-Instruct": { + "name": "Qwen2.5-32B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct", + "prompted": True, + "moe": False, + "size": 32, + "act_param": 32, + "open-data": "None", + }, + "Qwen/Qwen2.5-72B-Instruct": { + "name": "Qwen2.5-72B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct", + "prompted": True, + "moe": False, + "size": 72, + "act_param": 72, + "open-data": "None", + }, + "meta-llama/Llama-3.2-1B-Instruct": { + "name": "Llama-3.2-1B-Instruct", + "link": "https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct", + "prompted": True, + "moe": False, + "size": 1, + "act_param": 1, + "open-data": "None", + }, + "meta-llama/Llama-3.2-3B-Instruct": { + "name": "Llama-3.2-3B-Instruct", + "link": "https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct", + "prompted": True, + "moe": False, + "size": 3, + "act_param": 3, + "open-data": "None", + }, } \ No newline at end of file From 312321dd473c43ddeef3741375a1adcaf108c04a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:34:00 +0800 Subject: [PATCH 173/325] doc: update w/ 0.2.0 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a5ac62..9538ffa 100755 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@

## 📰 News +- **[2024-10-06]** We are releasing `bigcodebench==v0.2.0`! - **[2024-10-05]** We create a public code execution API on the [Hugging Face space](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator). - **[2024-10-01]** We have evaluated 139 models on BigCodeBench-Hard so far. Take a look at the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard)! - **[2024-08-19]** To make the evaluation fully reproducible, we add a real-time code execution session to the leaderboard. It can be viewed [here](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard). @@ -136,7 +137,7 @@ export GOOGLE_API_KEY= ## 💻 LLM-generated Code We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): -* See the attachment of our [v0.1.5](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.1.5). We include both `sanitized_samples.zip` and `sanitized_samples_calibrated.zip` for your convenience. +* See the attachment of our [v0.2.0](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.0). We include `sanitized_samples_calibrated.zip` for your convenience. ## Advanced Usage From b54dd6c7a0b6364fca0fa462c8344526d1d81f50 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 00:35:59 +0800 Subject: [PATCH 174/325] doc: minot update --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 9538ffa..68b1945 100755 --- a/README.md +++ b/README.md @@ -20,8 +20,7 @@ 🔥 Quick Start • 🚀 Remote Evaluation • 💻 LLM-generated Code • - 📜 Advanced Usage • - 🙏 Acknowledgement + 📜 Citation

## 📰 News From 8b9b46efee7f47054929140cef24e572ce7c393a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 16:02:12 +0800 Subject: [PATCH 175/325] doc: update link --- ADVANCED_USAGE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 428cf28..252abf1 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -71,7 +71,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--tokenizer_legacy`: Whether to use the legacy tokenizer, default to `False` - `--samples`: The path to the generated samples file, default to `None` - `--local_execute`: Whether to execute the samples locally, default to `False` -- `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://github.com/bigcode-project/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page. +- `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page. - `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` - `--save_pass_rate`: Whether to save the pass rate to a file, default to `True` - `--parallel`: The number of parallel processes, default to `None`, e.g. `--parallel 10` will evaluate 10 samples in parallel @@ -111,7 +111,7 @@ bigcodebench.generate \ ``` > -The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: +The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated.jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples: > ```bash From 991e41c06d54735d8ffd7e0651d1c080769039bb Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 16:34:57 +0800 Subject: [PATCH 176/325] doc: add warning --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 68b1945..25e61d7 100755 --- a/README.md +++ b/README.md @@ -85,6 +85,9 @@ pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcode ## 🚀 Remote Evaluation We use the greedy decoding as an example to show how to evaluate the generated code samples via remote API. +> [!Warning] +> +> To ease the generation, we use batch inference by default. However, the batch inference results could vary from *batch sizes to batch sizes* and *versions to versions*, at least for the vLLM backend. If you want to get more deterministic results for greedy decoding, please set `--bs` to `1`. > [!Note] > From 8cb06e4e4e9cf5070a625df246f0fa6a058c21d2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 7 Oct 2024 00:37:22 +0800 Subject: [PATCH 177/325] merge cfg --- setup.cfg | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index a9b7c74..4897f68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,9 +29,6 @@ install_requires = wget>=3.2 datasets gradio-client - -[options.extras_require] -generate = vllm numpy rich @@ -48,4 +45,4 @@ console_scripts = bigcodebench.syncheck = bigcodebench.syncheck:main bigcodebench.legacy_sanitize = bigcodebench.legacy_sanitize:main bigcodebench.generate = bigcodebench.generate:main - bigcodebench.inspect = bigcodebench.inspect:main \ No newline at end of file + bigcodebench.inspect = bigcodebench.inspect:main From 0a5154f1afba214de5a28946f115181ccc3bd9ae Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 7 Oct 2024 00:38:16 +0800 Subject: [PATCH 178/325] doc: merge installation cfg --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 25e61d7..4a6c70c 100755 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ To get started, please first set up the environment: ```bash # By default, you will use the remote evaluation API to execute the output samples. -pip install bigcodebench[generate] --upgrade +pip install bigcodebench --upgrade # You are suggested to use `flash-attn` for generating code samples. pip install packaging ninja @@ -75,7 +75,7 @@ pip install flash-attn --no-build-isolation ```bash # Install to use bigcodebench.generate -pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcodebench[generate]" --upgrade +pip install "git+https://github.com/bigcode-project/bigcodebench.git" --upgrade ``` From 1cb320fcb3b36945f9fd7ecec7e7f64cf492e5d9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 7 Oct 2024 00:40:15 +0800 Subject: [PATCH 179/325] Update ADVANCED_USAGE.md --- ADVANCED_USAGE.md | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 252abf1..ab6e0ae 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -3,25 +3,19 @@ To get started, please first set up the environment: ```bash -# Install to use bigcodebench.evaluate -pip install bigcodebench --upgrade -# If you want to use the evaluate locally, you need to install the requirements +# If you want to use the evaluate locally, you need to install the requirements in an isolated environment pip install -I -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt -# Install to use bigcodebench.generate -# You are strongly recommended to install the generate dependencies in a separate environment -pip install bigcodebench[generate] --upgrade +# You are strongly recommended to install the bigcodebench dependencies in another environment +pip install bigcodebench --upgrade ```
⏬ Install nightly version :: click to expand ::
```bash -# Install to use bigcodebench.evaluate +# Install to use bigcodebench pip install "git+https://github.com/bigcode-project/bigcodebench.git" --upgrade - -# Install to use bigcodebench.generate -pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcodebench[generate]" --upgrade ```
@@ -34,10 +28,8 @@ pip install "git+https://github.com/bigcode-project/bigcodebench.git#egg=bigcode git clone https://github.com/bigcode-project/bigcodebench.git cd bigcodebench export PYTHONPATH=$PYTHONPATH:$(pwd) -# Install to use bigcodebench.evaluate +# Install to use bigcodebench pip install -e . -# Install to use bigcodebench.generate -pip install -e .[generate] ``` From f6de469cb536920ac638345836cbacd11bd503b0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 16:54:19 +0000 Subject: [PATCH 180/325] docker: update Gradio.Dockerfile --- Docker/Gradio.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 9de820b..df4018f 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-b # upgrade to latest pip RUN pip install --upgrade pip -RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 huggingface-hub>=0.18.0 plotly python-dateutil==2.8.2 gradio-space-ci@git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio[oauth]==4.31.0 gradio_leaderboard==0.0.11 schedule==1.2.2 +RUN pip install APScheduler==3.10.1 black==23.11.0 click==8.1.3 huggingface-hub>=0.18.0 plotly python-dateutil==2.8.2 gradio-space-ci@git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.3 isort ruff gradio[oauth] schedule==1.2.2 # Add a new user "bigcodebenchuser" RUN adduser --disabled-password --gecos "" bigcodebenchuser From b8c1811623aae0493152f9dff01d65781f5102f7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 16:59:59 +0000 Subject: [PATCH 181/325] refactor(eval): update parallel default val --- bigcodebench/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index df8ad85..a082d56 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -119,7 +119,7 @@ def evaluate( remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/", pass_k: str = "1,5,10", save_pass_rate: bool = True, - parallel: int = None, + parallel: int = -1, min_time_limit: float = 1, max_as_limit: int = 30*1024, max_data_limit: int = 30*1024, @@ -167,7 +167,7 @@ def evaluate( pass_k = [int(k) for k in pass_k.split(",")] - if parallel is None: + if not parallel: n_workers = max(1, multiprocessing.cpu_count() // 2) else: n_workers = parallel From 0825835f4b86daf3f25e6bcf1786c71679cdba6e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 17:00:48 +0000 Subject: [PATCH 182/325] doc: update minimal full script --- run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run.sh b/run.sh index a84199e..c069e8e 100755 --- a/run.sh +++ b/run.sh @@ -9,5 +9,4 @@ bigcodebench.evaluate \ --model $MODEL \ --split $SPLIT \ --subset $SUBSET \ - --backend $BACKEND \ - --tp $NUM_GPU \ No newline at end of file + --backend $BACKEND \ No newline at end of file From dac3a0087a832aed84099f35705f93be2cb27119 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 17:01:56 +0000 Subject: [PATCH 183/325] doc: update parallel arg --- ADVANCED_USAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index ab6e0ae..1b2ca55 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -66,7 +66,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page. - `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` - `--save_pass_rate`: Whether to save the pass rate to a file, default to `True` -- `--parallel`: The number of parallel processes, default to `None`, e.g. `--parallel 10` will evaluate 10 samples in parallel +- `--parallel`: The number of parallel processes, default to `-1`, e.g. `--parallel 10` will evaluate 10 samples in parallel - `--min_time_limit`: The minimum time limit for the execution, default to `1`, e.g. `--min_time_limit 10` will evaluate the samples with at least 10 seconds - `--max_as_limit`: The maximum address space limit for the execution, default to `30*1024` (30 GB), e.g. `--max_as_limit 20*1024` will evaluate the samples with at most 20 GB - `--max_data_limit`: The maximum data segment limit for the execution, default to `30*1024` (30 GB), e.g. `--max_data_limit 20*1024` will evaluate the samples with at most 20 GB From 58b3f2d01b285e5db9c2493114f4747dfb883dcc Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 6 Oct 2024 18:52:59 +0000 Subject: [PATCH 184/325] fix: change parallel logic --- bigcodebench/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index a082d56..5a9fab8 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -167,7 +167,7 @@ def evaluate( pass_k = [int(k) for k in pass_k.split(",")] - if not parallel: + if parallel < 1: n_workers = max(1, multiprocessing.cpu_count() // 2) else: n_workers = parallel From 2a28c61ccfdcecf0f3466b955b5a0b1028512310 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 7 Oct 2024 04:39:33 +0800 Subject: [PATCH 185/325] doc: update model outputs link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a6c70c..c6ca7b7 100755 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ export GOOGLE_API_KEY= ## 💻 LLM-generated Code We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): -* See the attachment of our [v0.2.0](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.0). We include `sanitized_samples_calibrated.zip` for your convenience. +* See the attachment of our [v0.2.0.post3](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.0.post3). We include `sanitized_samples_calibrated.zip` for your convenience. ## Advanced Usage From 817e63b25fe38a0f97abbfcc97fc8ed2e08b2474 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 7 Oct 2024 11:39:47 +0800 Subject: [PATCH 186/325] doc: benchmark description --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index c6ca7b7..ab3747b 100755 --- a/README.md +++ b/README.md @@ -48,6 +48,10 @@ BigCodeBench is an **_easy-to-use_** benchmark for solving **_practical_** and **_challenging_** tasks via code. It aims to evaluate the true programming capabilities of large language models (LLMs) in a more realistic setting. The benchmark is designed for HumanEval-like function-level code generation tasks, but with much more complex instructions and diverse function calls. +There are two splits in BigCodeBench: +- `Complete`: Thes split is designed for code completion based on the comprehensive docstrings. +- `Instruct`: The split works for the instruction-tuned and chat models only, where the models are asked to generate a code snippet based on the natural language instructions. The instructions only contain necessary information, and require more complex reasoning. + ### Why BigCodeBench? BigCodeBench focuses on task automation via code generation with *diverse function calls* and *complex instructions*, with: From 112623038c349d4755f85f8b05a8bceeae1886df Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 8 Oct 2024 16:01:42 +0800 Subject: [PATCH 187/325] remove reflection model --- analysis/utils.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index ce81bd6..87453fd 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1133,24 +1133,24 @@ "act_param": 9, "open-data": "None", }, - "mattshumer/ref_70_e3_prefill": { - "name": "Reflection-Llama-3.1-70B", - "link": "https://huggingface.co/mattshumer/ref_70_e3", - "prompted": True, - "moe": False, - "size": 70, - "act_param": 70, - "open-data": "None", - }, - "mattshumer/ref_70_e3": { - "name": "Reflection-Llama-3.1-70B (Recommended Settings)", - "link": "https://huggingface.co/mattshumer/ref_70_e3", - "prompted": True, - "moe": False, - "size": 70, - "act_param": 70, - "open-data": "None", - }, + # "mattshumer/ref_70_e3_prefill": { + # "name": "Reflection-Llama-3.1-70B", + # "link": "https://huggingface.co/mattshumer/ref_70_e3", + # "prompted": True, + # "moe": False, + # "size": 70, + # "act_param": 70, + # "open-data": "None", + # }, + # "mattshumer/ref_70_e3": { + # "name": "Reflection-Llama-3.1-70B (Recommended Settings)", + # "link": "https://huggingface.co/mattshumer/ref_70_e3", + # "prompted": True, + # "moe": False, + # "size": 70, + # "act_param": 70, + # "open-data": "None", + # }, "o1-preview-2024-09-12": { "name": "o1-Preview-2024-09-12 (temperature=1)", "link": "https://o1.ai/o1-preview", From e35d6257f149981fbc237abbb74493c14018e8ed Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 13 Oct 2024 03:39:01 +0000 Subject: [PATCH 188/325] doc: add impact --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index ab3747b..dba0bd6 100755 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@

+ 💥 Impact • 📰 News • 🔥 Quick Start • 🚀 Remote Evaluation • @@ -23,6 +24,17 @@ 📜 Citation

+## 💥 Impact +BigCodeBench has been used by the many LLM teams including: +- Zhipu AI +- Alibaba Qwen +- DeepSeek +- Amazon AWS +- Snowflake AI Research +- ServiceNow Research +- Meta AI +- Cohere AI + ## 📰 News - **[2024-10-06]** We are releasing `bigcodebench==v0.2.0`! - **[2024-10-05]** We create a public code execution API on the [Hugging Face space](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator). From 3b4a058d0768ec1b9d737f1a4ad2de52e4b3f7f1 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 14 Oct 2024 17:00:58 +0800 Subject: [PATCH 189/325] fix(doc): typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dba0bd6..edb220e 100755 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@

## 💥 Impact -BigCodeBench has been used by the many LLM teams including: +BigCodeBench has been used by many LLM teams including: - Zhipu AI - Alibaba Qwen - DeepSeek From e10d361a8dde5d59724a5604b1fab4c65ff735c9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 18 Oct 2024 00:03:02 +0800 Subject: [PATCH 190/325] docs: update impact --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index edb220e..0c1958f 100755 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ BigCodeBench has been used by many LLM teams including: - Zhipu AI - Alibaba Qwen - DeepSeek -- Amazon AWS +- Amazon AWS AI - Snowflake AI Research - ServiceNow Research - Meta AI From e25440ea1b8cd20b83c9f2881f492f33b92fc898 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 31 Oct 2024 02:13:02 +0800 Subject: [PATCH 191/325] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0c1958f..c37b203 100755 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ BigCodeBench has been used by many LLM teams including: - ServiceNow Research - Meta AI - Cohere AI +- Sakana AI ## 📰 News - **[2024-10-06]** We are releasing `bigcodebench==v0.2.0`! From c5a22bff708b43698630da8d03dfe5063f52216c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 1 Nov 2024 17:21:31 +0800 Subject: [PATCH 192/325] feat(codegen): support model revision --- bigcodebench/generate.py | 4 +++- bigcodebench/provider/__init__.py | 4 ++++ bigcodebench/provider/base.py | 2 ++ bigcodebench/provider/hf.py | 1 + bigcodebench/provider/vllm.py | 1 + 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 6333261..58f1ab7 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -139,6 +139,7 @@ def run_codegen( backend: str = "vllm", base_url: str = None, tp: int = 1, + revision: str = "main", trust_remote_code: bool = False, tokenizer_name: str = None, tokenizer_legacy: bool = False, @@ -173,6 +174,7 @@ def run_codegen( response_prefix=response_prefix, base_url=base_url, tp=tp, + revision=revision, trust_remote_code=trust_remote_code, direct_completion=direct_completion, tokenizer_name=tokenizer_name, @@ -180,7 +182,7 @@ def run_codegen( ) extra = "-" + subset if subset != "full" else "" - identifier = model.replace("/", "--") + f"--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" + identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" target_path = os.path.join(root, identifier) diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index 67123f9..d519124 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -12,6 +12,8 @@ def make_model( # instruction model only instruction_prefix: str = None, response_prefix: str = None, + # vllm and hf only + revision: str = "main", # vllm only tp: int = 1, direct_completion: bool = False, @@ -32,6 +34,7 @@ def make_model( split=split, temperature=temperature, max_new_tokens=max_new_tokens, + revision=revision, dataset=dataset, direct_completion=direct_completion, tp=tp, @@ -47,6 +50,7 @@ def make_model( split=split, temperature=temperature, max_new_tokens=max_new_tokens, + revision=revision, dataset=dataset, direct_completion=direct_completion, instruction_prefix=instruction_prefix, diff --git a/bigcodebench/provider/base.py b/bigcodebench/provider/base.py index ebec843..5a24b59 100644 --- a/bigcodebench/provider/base.py +++ b/bigcodebench/provider/base.py @@ -12,6 +12,7 @@ def __init__( split: str, temperature: float = 0.8, max_new_tokens: int = 1280, + revision: str = "main", dtype: str = "bfloat16", # default direct_completion: bool = False, trust_remote_code: bool = False, @@ -29,6 +30,7 @@ def __init__( self.skip_special_tokens = False self.max_new_tokens = max_new_tokens self.dtype = dtype + self.revision = revision self.direct_completion = direct_completion self.trust_remote_code = trust_remote_code self.tokenizer_name = tokenizer_name diff --git a/bigcodebench/provider/hf.py b/bigcodebench/provider/hf.py index c3136c8..a85957d 100644 --- a/bigcodebench/provider/hf.py +++ b/bigcodebench/provider/hf.py @@ -27,6 +27,7 @@ def __init__( "trust_remote_code": self.trust_remote_code, "torch_dtype": getattr(torch, self.dtype), "attn_implementation": attn_implementation, # "eager", "flash_attention_2", "sdpa" + "revision": self.revision, } self.skip_special_tokens = True diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 3d0aaf4..171a41c 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -18,6 +18,7 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: "tensor_parallel_size": int(os.getenv("VLLM_N_GPUS", tp)), "dtype": self.dtype, "trust_remote_code": self.trust_remote_code, + "revision": self.revision, } if self.tokenizer_name is None: self.tokenizer_name = self.name From a810f315287cc0a7860791db79b9967420dba80e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 1 Nov 2024 17:24:20 +0800 Subject: [PATCH 193/325] doc: add model revision --- ADVANCED_USAGE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 1b2ca55..67fe359 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -57,6 +57,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--id_range`: The range of the tasks to evaluate, default to `None`, e.g. `--id_range 10,20` will evaluate the tasks from 10 to 20 - `--backend`: The backend to use, default to `vllm` - `--base_url`: The base URL of the backend for OpenAI-compatible APIs, default to `None` +- `--revision`: The revision of the model with the vLLM or HF backend, default to `main` - `--tp`: The tensor parallel size for the vLLM backend, default to `1` - `--trust_remote_code`: Whether to trust the remote code, default to `False` - `--tokenizer_name`: The name of the customized tokenizer, default to `None` From e8798f47a2c43ada4625f9b67c7db8436980d9f9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 2 Nov 2024 17:22:46 +0800 Subject: [PATCH 194/325] fix: change id_range type --- bigcodebench/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 58f1ab7..6b3fe37 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -135,7 +135,7 @@ def run_codegen( strip_newlines: bool = False, direct_completion: bool = False, resume: bool = True, - id_range: Tuple[int, int] = None, + id_range: str = None, backend: str = "vllm", base_url: str = None, tp: int = 1, @@ -152,6 +152,7 @@ def run_codegen( print("Greedy decoding ON (--greedy): setting n_samples=1, temperature=0") if id_range is not None: + id_range = [int(i) for i in id_range.split("-")] assert len(id_range) == 2, "id_range must be a list of length 2" assert id_range[0] < id_range[1], "id_range must be increasing" id_range = tuple(id_range) From 216543126cc8fafa9e7171f5546c38ec801b4a6c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 2 Nov 2024 19:31:26 +0800 Subject: [PATCH 195/325] fix(codegen): stop by upper bound --- bigcodebench/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 6b3fe37..9f29cad 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -64,7 +64,7 @@ def codegen( if id_num < low: p.console.print(f"Skipping {task_id} as it is not in {id_range}") continue - if id_num > id_range[1]: + if id_num >= id_range[1]: break p_name = task_id.replace("/", "_") From 1e243647dfe02b7ce44ad4a3237cfc81fc4d5e69 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 5 Nov 2024 03:09:00 +0800 Subject: [PATCH 196/325] feat: using datasets to load --- bigcodebench/data/bigcodebench.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index da2ad5d..9a3ee9d 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -26,14 +26,8 @@ def _ready_bigcodebench_path(subset="full", version="default") -> str: ) extra = "-" + subset if subset != "full" else "" - - try: - dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) - make_cache(url, dataset, path) - except: - if os.path.exists(path): - os.remove(path) - make_cache(url, None, path, gh=True) + dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION) + make_cache(url, dataset, path) return path From cb283fdd11d0f8abaaf5e7eec59641b52ccf13fc Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 5 Nov 2024 03:12:45 +0800 Subject: [PATCH 197/325] feat: customize instruction and response --- bigcodebench/generate.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 9f29cad..757b08c 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -139,6 +139,8 @@ def run_codegen( backend: str = "vllm", base_url: str = None, tp: int = 1, + instruction_prefix: str = None, + response_prefix: str = None, revision: str = "main", trust_remote_code: bool = False, tokenizer_name: str = None, @@ -160,8 +162,10 @@ def run_codegen( # Make project dir os.makedirs(root, exist_ok=True) - instruction_prefix = "Please provide a self-contained Python script that solves the following problem in a markdown code block:" - response_prefix = "Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:" + if instruction_prefix is None: + instruction_prefix = "Please provide a self-contained Python script that solves the following problem in a markdown code block:" + if response_prefix is None: + response_prefix = "Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:" # Make dir for codes generated by each model model_runner = make_model( From 974e67918e36ee923b06808e5a2edfaa8d7c9319 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 5 Nov 2024 03:20:51 +0800 Subject: [PATCH 198/325] fix: make google api do n samples --- bigcodebench/gen/util/google_request.py | 8 ++++++-- bigcodebench/provider/google.py | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bigcodebench/gen/util/google_request.py b/bigcodebench/gen/util/google_request.py index 8a88842..7ce935b 100644 --- a/bigcodebench/gen/util/google_request.py +++ b/bigcodebench/gen/util/google_request.py @@ -5,13 +5,17 @@ def make_request( - client: genai.GenerativeModel, temperature, messages, max_new_tokens=2048 + client: genai.GenerativeModel, + messages: List, + temperature: float, + n: int, + max_new_tokens: int = 2048, ) -> genai.types.GenerateContentResponse: messages = [{"role": m["role"], "parts": [m["content"]]} for m in messages] response = client.generate_content( messages, generation_config=genai.types.GenerationConfig( - candidate_count=1, + candidate_count=n, max_output_tokens=max_new_tokens, temperature=temperature, ), diff --git a/bigcodebench/provider/google.py b/bigcodebench/provider/google.py index 0cd5416..c9781ca 100644 --- a/bigcodebench/provider/google.py +++ b/bigcodebench/provider/google.py @@ -36,10 +36,9 @@ def codegen( ret = make_auto_request( self.client, message, - self.name, n=num_samples, - max_tokens=self.max_new_tokens, temperature=self.temperature, + max_new_tokens=self.max_new_tokens, ) for candidate in ret.candidates: parts = candidate.content.parts From 492080811f468758e69ca279489fc729783715bd Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 5 Nov 2024 03:56:25 +0800 Subject: [PATCH 199/325] feat: change google api request --- bigcodebench/gen/util/google_request.py | 5 ++--- bigcodebench/provider/google.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bigcodebench/gen/util/google_request.py b/bigcodebench/gen/util/google_request.py index 7ce935b..6517650 100644 --- a/bigcodebench/gen/util/google_request.py +++ b/bigcodebench/gen/util/google_request.py @@ -6,14 +6,13 @@ def make_request( client: genai.GenerativeModel, - messages: List, + message: str, temperature: float, n: int, max_new_tokens: int = 2048, ) -> genai.types.GenerateContentResponse: - messages = [{"role": m["role"], "parts": [m["content"]]} for m in messages] response = client.generate_content( - messages, + [{"role": "user", "parts": [message]}], generation_config=genai.types.GenerationConfig( candidate_count=n, max_output_tokens=max_new_tokens, diff --git a/bigcodebench/provider/google.py b/bigcodebench/provider/google.py index c9781ca..2194c47 100644 --- a/bigcodebench/provider/google.py +++ b/bigcodebench/provider/google.py @@ -35,7 +35,7 @@ def codegen( ) ret = make_auto_request( self.client, - message, + message=message, n=num_samples, temperature=self.temperature, max_new_tokens=self.max_new_tokens, From 1d9ea6af233cf8e86ccf279467b0f8e2b4c93122 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 5 Nov 2024 03:56:55 +0800 Subject: [PATCH 200/325] feat: batch o1 and deepseek-chat via concurrency --- bigcodebench/gen/util/openai_request.py | 30 +++------- bigcodebench/provider/openai.py | 75 ++++++++++++++++++++----- bigcodebench/provider/utility.py | 9 ++- 3 files changed, 77 insertions(+), 37 deletions(-) diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index e347ffe..a745d8d 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -1,4 +1,3 @@ -import signal import time import openai @@ -14,53 +13,38 @@ def make_request( n: int = 1, **kwargs ) -> ChatCompletion: - system_msg = "You are a helpful assistant good at coding." - if ( - kwargs.get("response_format", None) - and kwargs["response_format"]["type"] == "json_object" - ): - system_msg = "You are a helpful assistant designed to output JSON." - + kwargs["top_p"] = 0.95 + kwargs["max_completion_tokens"] = max_tokens + if model.startswith("o1-"): # pop top-p and max_completion_tokens + kwargs.pop("top_p") + kwargs.pop("max_completion_tokens") + return client.chat.completions.create( model=model, messages=[ - {"role": "system", "content": system_msg}, {"role": "user", "content": message}, ], - max_tokens=max_tokens, temperature=temperature, n=n, **kwargs ) -def handler(signum, frame): - # swallow signum and frame - raise Exception("end of time") - - def make_auto_request(*args, **kwargs) -> ChatCompletion: ret = None while ret is None: try: - signal.signal(signal.SIGALRM, handler) - signal.alarm(100) ret = make_request(*args, **kwargs) - signal.alarm(0) except openai.RateLimitError: print("Rate limit exceeded. Waiting...") - signal.alarm(0) time.sleep(5) except openai.APIConnectionError: print("API connection error. Waiting...") - signal.alarm(0) time.sleep(5) except openai.APIError as e: print(e) - signal.alarm(0) except Exception as e: print("Unknown error. Waiting...") print(e) - signal.alarm(0) time.sleep(1) - return ret + return ret \ No newline at end of file diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 9eba02e..76e315e 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -1,12 +1,12 @@ import os from typing import List -from tqdm import tqdm import openai -from bigcodebench.provider.base import DecoderBase from bigcodebench.gen.util.openai_request import make_auto_request from bigcodebench.provider.utility import make_raw_chat_prompt +from bigcodebench.provider.base import DecoderBase +from bigcodebench.provider.utility import concurrent_call class OpenAIChatDecoder(DecoderBase): def __init__(self, name: str, base_url=None, **kwargs) -> None: @@ -15,34 +15,83 @@ def __init__(self, name: str, base_url=None, **kwargs) -> None: api_key=os.getenv("OPENAI_API_KEY", "none"), base_url=base_url ) + # def codegen( + # self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + # ) -> List[str]: + # if do_sample: + # assert self.temperature > 0, "Temperature must be positive for sampling" + # all_outputs = [] + # for prompt in tqdm(prompts): + # outputs = [] + # message = make_raw_chat_prompt( + # task_prompt=prompt, + # subset=self.subset, + # split=self.split, + # instruction_prefix=self.instruction_prefix, + # response_prefix=self.response_prefix, + # tokenizer=None, + # ) + # ret = make_auto_request( + # self.client, + # message=message, + # model=self.name, + # max_tokens=self.max_new_tokens, + # temperature=self.temperature, + # n=num_samples, + # ) + # for item in ret.choices: + # outputs.append(item.message.content) + # all_outputs.append(outputs) + # return all_outputs + + # def is_direct_completion(self) -> bool: + # return False + def codegen( self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 ) -> List[str]: if do_sample: assert self.temperature > 0, "Temperature must be positive for sampling" + messages = [make_raw_chat_prompt( + task_prompt=prompt, + subset=self.subset, + split=self.split, + instruction_prefix=self.instruction_prefix, + response_prefix=self.response_prefix, + tokenizer=None, + ) for prompt in prompts] + # use concurrency based batching for o1 and deepseek models + if self.name.startswith("o1-") or self.name == "deepseek-chat": + return self._codegen_batch_via_concurrency(messages, num_samples) + + return self._codegen_api_batch(messages, num_samples) + + def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str]: + client = openai.OpenAI( + api_key=os.getenv("OPENAI_API_KEY", "none"), base_url=self.base_url + ) + all_outputs = [] - for prompt in tqdm(prompts): - outputs = [] - message = make_raw_chat_prompt( - task_prompt=prompt, - subset=self.subset, - split=self.split, - instruction_prefix=self.instruction_prefix, - response_prefix=self.response_prefix, - tokenizer=None, - ) + for message in messages: ret = make_auto_request( - self.client, + client, message=message, model=self.name, max_tokens=self.max_new_tokens, temperature=self.temperature, n=num_samples, ) + outputs = [] for item in ret.choices: outputs.append(item.message.content) all_outputs.append(outputs) return all_outputs + def _codegen_batch_via_concurrency(self, messages: List[str], num_samples: int) -> List[str]: + batches = concurrent_call( + num_samples, self._codegen_api_batch, messages, num_samples=1 + ) + return [b[0] for b in batches] + def is_direct_completion(self) -> bool: return False \ No newline at end of file diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py index 60a00e5..bb27539 100644 --- a/bigcodebench/provider/utility.py +++ b/bigcodebench/provider/utility.py @@ -1,5 +1,6 @@ from typing import List from transformers import AutoTokenizer +from concurrent.futures import ThreadPoolExecutor EOS = [ "<|endoftext|>", @@ -64,4 +65,10 @@ def make_raw_chat_prompt( ], tokenize=False, ).split(_MAGIC_SPLITTER_)[0] - return task_prompt \ No newline at end of file + return task_prompt + + +def concurrent_call(n, callback, /, *args, **kwargs): + with ThreadPoolExecutor(max_workers=n) as executor: + futures = [executor.submit(callback, *args, **kwargs) for _ in range(n)] + return [future.result() for future in futures] \ No newline at end of file From 813712f9220d0532f36757fee39ed841da2312d9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 5 Nov 2024 05:29:33 +0800 Subject: [PATCH 201/325] feat: add 3.5 haiku and grok beta --- analysis/utils.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/analysis/utils.py b/analysis/utils.py index 87453fd..4cd9862 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1277,4 +1277,58 @@ "act_param": 3, "open-data": "None", }, -} \ No newline at end of file + "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { + "name": "Llama-3.1-Nemotron-70B-Instruct", + "link": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "Partial", + }, + "claude-3-5-sonnet-20241022": { + "name": "Claude-3.5-Sonnet-20241022", + "link": "https://claude.ai/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "ibm-granite/granite-3.0-8b-instruct": { + "name": "Granite-3.0-8B-Instruct", + "link": "https://huggingface.co/ibm-granite/granite-3.0-8b-instruct", + "prompted": True, + "moe": False, + "size": 8, + "act_param": 8, + "open-data": "None", + }, + "ibm-granite/granite-3.0-2b-instruct": { + "name": "Granite-3.0-2B-Instruct", + "link": "https://huggingface.co/ibm-granite/granite-3.0-2b-instruct", + "prompted": True, + "moe": False, + "size": 2, + "act_param": 2, + "open-data": "None", + }, + "grok-beta--main": { + "name": "Grok-Beta", + "link": "https://grok.com/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "claude-3-5-haiku-20241022--main": { + "name": "Claude-3.5-Haiku-20241022", + "link": "https://claude.ai/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, +} From 16ec422e9af5c9f6663bdca737cce4d8460647a5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 9 Nov 2024 01:16:07 +0800 Subject: [PATCH 202/325] fix(evaluate): update the calibration setup --- bigcodebench/evaluate.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 5a9fab8..44c7f93 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -233,7 +233,7 @@ def evaluate( if "solution" in sample else problems[task_id]["complete_prompt"] + sample["completion"] ) - if "sanitized-calibrated" in samples: + if "sanitized_calibrated" in samples: solution = problems[task_id]["code_prompt"] + "\n pass\n" + solution remainings.add(sample["_identifier"]) args = ( @@ -254,22 +254,22 @@ def evaluate( assert n_samples == len(remainings), "Missing problems in unfinished" assert len(completion_id) == len(problems), "Missing problems in samples" - def stucking_checker(): - while remainings: - last_size = len(remainings) - time.sleep(240) - if last_size != len(remainings) or len(remainings) == 0: - continue - # Potential stucking - warn("No samples had finished testing in the last 240s") - warn(f"{len(remainings)} samples to be tested: {remainings}") + def stucking_checker(): + while remainings: + last_size = len(remainings) + time.sleep(240) + if last_size != len(remainings) or len(remainings) == 0: + continue + # Potential stucking + warn("No samples had finished testing in the last 240s") + warn(f"{len(remainings)} samples to be tested: {remainings}") - threading.Thread(target=stucking_checker).start() + threading.Thread(target=stucking_checker).start() - for future in tqdm(as_completed(futures), total=n_samples): - result = future.result() - remainings.remove(result["_identifier"]) - eval_results[result["task_id"]].append(result) + for future in tqdm(as_completed(futures), total=n_samples): + result = future.result() + remainings.remove(result["_identifier"]) + eval_results[result["task_id"]].append(result) # sort the results for each problem by completion_id for task_id, task_results in eval_results.items(): @@ -307,7 +307,7 @@ def stucking_checker(): pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0] pass_at_k["split"] = split pass_at_k["subset"] = subset - pass_at_k["calibrated"] = "sanitized-calibrated" in samples + pass_at_k["calibrated"] = "sanitized_calibrated" in samples pass_at_k["gt_pass_rate"] = gt_pass_rate pass_at_k["failed_tasks"] = failed_tasks From 570a4c8f783f1c954e2256bf6d25e89c2e4cd0ea Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 9 Nov 2024 17:12:19 +0800 Subject: [PATCH 203/325] feat(evaluate): add no_execute flag --- bigcodebench/evaluate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 44c7f93..6d02b4b 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -115,6 +115,7 @@ def evaluate( split: str, subset: str, samples: Optional[str] = None, + no_execute: bool = False, local_execute: bool = False, remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/", pass_k: str = "1,5,10", @@ -135,6 +136,10 @@ def evaluate( subset=subset, **model_kwargs, ) + + if no_execute: + return + assert samples is not None, "No samples provided" if os.path.isdir(samples): From 9ff42caca16b461b8eb5b5d74a371fe4f38c0ad9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 9 Nov 2024 17:15:52 +0800 Subject: [PATCH 204/325] fix(doc): change id_range input --- ADVANCED_USAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 67fe359..0cd8007 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -54,7 +54,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--strip_newlines`: Whether to strip newlines, default to `False`, set to `True` to strip newlines for some model series like StarCoder2 - `--direct_completion`: Whether to use direct completion, default to `False` - `--resume`: Whether to resume the evaluation, default to `True`, set to `False` to re-run the evaluation -- `--id_range`: The range of the tasks to evaluate, default to `None`, e.g. `--id_range 10,20` will evaluate the tasks from 10 to 20 +- `--id_range`: The range of the tasks to evaluate, default to `None`, e.g. `--id_range 10-20` will evaluate the tasks from 10 to 20 - `--backend`: The backend to use, default to `vllm` - `--base_url`: The base URL of the backend for OpenAI-compatible APIs, default to `None` - `--revision`: The revision of the model with the vLLM or HF backend, default to `main` From 8ed15f69c38b3f3d2c0b0ddf8bf638170af9aeba Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 9 Nov 2024 17:52:10 +0800 Subject: [PATCH 205/325] fix(codegen): update make_request --- bigcodebench/gen/util/google_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/gen/util/google_request.py b/bigcodebench/gen/util/google_request.py index 6517650..8e696b4 100644 --- a/bigcodebench/gen/util/google_request.py +++ b/bigcodebench/gen/util/google_request.py @@ -26,7 +26,7 @@ def make_request( ], ) - return response.text + return response def make_auto_request(*args, **kwargs) -> genai.types.GenerateContentResponse: From 0f4df3e764e9fa132374fbaa206d3caa060219d0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 9 Nov 2024 17:52:44 +0800 Subject: [PATCH 206/325] fix(codegen): remove commented code --- bigcodebench/provider/openai.py | 36 +-------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 76e315e..91c1882 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -11,41 +11,7 @@ class OpenAIChatDecoder(DecoderBase): def __init__(self, name: str, base_url=None, **kwargs) -> None: super().__init__(name, **kwargs) - self.client = openai.OpenAI( - api_key=os.getenv("OPENAI_API_KEY", "none"), base_url=base_url - ) - - # def codegen( - # self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 - # ) -> List[str]: - # if do_sample: - # assert self.temperature > 0, "Temperature must be positive for sampling" - # all_outputs = [] - # for prompt in tqdm(prompts): - # outputs = [] - # message = make_raw_chat_prompt( - # task_prompt=prompt, - # subset=self.subset, - # split=self.split, - # instruction_prefix=self.instruction_prefix, - # response_prefix=self.response_prefix, - # tokenizer=None, - # ) - # ret = make_auto_request( - # self.client, - # message=message, - # model=self.name, - # max_tokens=self.max_new_tokens, - # temperature=self.temperature, - # n=num_samples, - # ) - # for item in ret.choices: - # outputs.append(item.message.content) - # all_outputs.append(outputs) - # return all_outputs - - # def is_direct_completion(self) -> bool: - # return False + self.base_url = base_url def codegen( self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 From d40eceb157ce030755d211412fd01f4f08e3df98 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 9 Nov 2024 17:54:19 +0800 Subject: [PATCH 207/325] doc: add params --- ADVANCED_USAGE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 0cd8007..0b2bf7b 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -57,12 +57,15 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--id_range`: The range of the tasks to evaluate, default to `None`, e.g. `--id_range 10-20` will evaluate the tasks from 10 to 20 - `--backend`: The backend to use, default to `vllm` - `--base_url`: The base URL of the backend for OpenAI-compatible APIs, default to `None` +- `--instruction_prefix`: The instruction prefix for the Anthropic backend, default to `None` +- `--response_prefix`: The response prefix for the Anthropic backend, default to `None` - `--revision`: The revision of the model with the vLLM or HF backend, default to `main` - `--tp`: The tensor parallel size for the vLLM backend, default to `1` - `--trust_remote_code`: Whether to trust the remote code, default to `False` - `--tokenizer_name`: The name of the customized tokenizer, default to `None` - `--tokenizer_legacy`: Whether to use the legacy tokenizer, default to `False` - `--samples`: The path to the generated samples file, default to `None` +- `--no_execute`: Whether to not execute the samples, default to `False` - `--local_execute`: Whether to execute the samples locally, default to `False` - `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page. - `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` From e517a9e2e99e262cf3c464332c6ee0afbbe872d0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 11 Nov 2024 18:10:01 +0800 Subject: [PATCH 208/325] fix(evaluate): update backup pass_k result path --- bigcodebench/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 6d02b4b..590d1ae 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -370,7 +370,7 @@ def stucking_checker(): print(f"Save pass@k to {pass_at_k_path}? [Y/N]") decision = input() if decision.lower() == "y": - new_path = result_path + ".bak" + new_path = pass_at_k_path + ".bak" while os.path.isfile(new_path): new_path += ".bak" os.rename(pass_at_k_path, new_path) From 54794ed1510959df76dcad34fa50689e6ff9c666 Mon Sep 17 00:00:00 2001 From: LRL Date: Tue, 12 Nov 2024 11:40:05 +0800 Subject: [PATCH 209/325] fix missing trust_remote_code parameter --- bigcodebench/provider/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index d519124..ff27a91 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -40,6 +40,7 @@ def make_model( tp=tp, instruction_prefix=instruction_prefix, response_prefix=response_prefix, + trust_remote_code=trust_remote_code, ) elif backend == "hf": from bigcodebench.provider.hf import HuggingFaceDecoder @@ -56,6 +57,7 @@ def make_model( instruction_prefix=instruction_prefix, response_prefix=response_prefix, attn_implementation=attn_implementation, + trust_remote_code=trust_remote_code, ) elif backend == "openai": from bigcodebench.provider.openai import OpenAIChatDecoder From 864586393ef9e11e0d09d8e9a58f1d7c632e75f4 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 12 Nov 2024 17:19:15 +0800 Subject: [PATCH 210/325] fix: add tokenizer customization back --- bigcodebench/provider/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index ff27a91..ef19f4e 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -41,6 +41,8 @@ def make_model( instruction_prefix=instruction_prefix, response_prefix=response_prefix, trust_remote_code=trust_remote_code, + tokenizer_name=tokenizer_name, + tokenizer_legacy=tokenizer_legacy, ) elif backend == "hf": from bigcodebench.provider.hf import HuggingFaceDecoder @@ -58,6 +60,8 @@ def make_model( response_prefix=response_prefix, attn_implementation=attn_implementation, trust_remote_code=trust_remote_code, + tokenizer_name=tokenizer_name, + tokenizer_legacy=tokenizer_legacy, ) elif backend == "openai": from bigcodebench.provider.openai import OpenAIChatDecoder From 9d7af5431bc0aa7a0ca55521e31bf05c2456f687 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 12 Nov 2024 23:05:05 +0800 Subject: [PATCH 211/325] add doc for result submission --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c37b203..debba10 100755 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ 🔥 Quick Start • 🚀 Remote Evaluation • 💻 LLM-generated Code • + 🧑 Advanced Usage • + 📰 Result Submission • 📜 Citation

@@ -158,10 +160,14 @@ export GOOGLE_API_KEY= We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): * See the attachment of our [v0.2.0.post3](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.0.post3). We include `sanitized_samples_calibrated.zip` for your convenience. -## Advanced Usage +## 🧑 Advanced Usage Please refer to the [ADVANCED USAGE](https://github.com/bigcode-project/bigcodebench/blob/main/ADVANCED_USAGE.md) for more details. +## 📰 Result Submission + +Please email both the generated code samples and the execution results to [terry.zhuo@monash.edu](mailto:terry.zhuo@monash.edu) if you would like to contribute your model to the leaderboard. Note that the file names should be in the format of `[model_name]--[revision]--[bigcodebench|bigcodebench-hard]-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated.jsonl` and `[model_name]--[revision]--[bigcodebench|bigcodebench-hard]-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_eval_results.json`. You can [file an issue](https://github.com/bigcode-project/bigcodebench/issues/new/choose) to remind us if we do not respond to your email within 3 days. + ## 📜 Citation ```bibtex From 13f07c9e5ae6d8da689c7d793112c827df1863cb Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 17 Nov 2024 00:55:07 +0800 Subject: [PATCH 212/325] feat(codegen): add the progress bar for openai API --- bigcodebench/provider/openai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 91c1882..52f8a05 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -1,6 +1,6 @@ import os from typing import List - +from tqdm import tqdm import openai from bigcodebench.gen.util.openai_request import make_auto_request @@ -38,7 +38,7 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] ) all_outputs = [] - for message in messages: + for message in tqdm(messages): ret = make_auto_request( client, message=message, From b888ce6835a11b865d36d1103da6799266775c6c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 17 Nov 2024 01:03:17 +0800 Subject: [PATCH 213/325] feat(evaluate): add backoff for file reading --- bigcodebench/evaluate.py | 46 ++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 590d1ae..ec50ace 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -12,6 +12,7 @@ from warnings import warn from gradio_client import Client, handle_file +import httpx import numpy as np from termcolor import cprint from tqdm import tqdm @@ -149,22 +150,27 @@ def evaluate( result_path = samples.replace(".jsonl", "_eval_results.json") if not local_execute: - - client = Client(remote_execute_api) - results, pass_at_k = client.predict( - split=split, - subset=subset, - samples=handle_file(samples), - pass_k=pass_k, - parallel=parallel, - min_time_limit=min_time_limit, - max_as_limit=max_as_limit, - max_data_limit=max_data_limit, - max_stack_limit=max_stack_limit, - check_gt_only=check_gt_only, - no_gt=no_gt, - api_name="/predict" - ) + while True: + try: + client = Client(remote_execute_api) + results, pass_at_k = client.predict( + split=split, + subset=subset, + samples=handle_file(samples), + pass_k=pass_k, + parallel=parallel, + min_time_limit=min_time_limit, + max_as_limit=max_as_limit, + max_data_limit=max_data_limit, + max_stack_limit=max_stack_limit, + check_gt_only=check_gt_only, + no_gt=no_gt, + api_name="/predict" + ) + break + except httpx.ReadTimeout: + print("Read timeout error. Retrying in 4s...") + time.sleep(4) gt_pass_rate = pass_at_k["gt_pass_rate"] failed_tasks = pass_at_k["failed_tasks"] @@ -388,3 +394,11 @@ def main(): if __name__ == "__main__": main() + +def main(): + from fire import Fire + + Fire(evaluate) + +if __name__ == "__main__": + main() From b9a7b1786975199b1062e4cb38e97397d47ae31e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 20 Nov 2024 01:55:36 +0800 Subject: [PATCH 214/325] feat(evaluate): add backoff for concurrent issues --- bigcodebench/evaluate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index ec50ace..4b00655 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -7,6 +7,7 @@ import time from collections import Counter, defaultdict from concurrent.futures import ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED +from concurrent.futures._base import CancelledError from datetime import datetime from typing import Any, Dict, List, Tuple, Optional from warnings import warn @@ -168,7 +169,7 @@ def evaluate( api_name="/predict" ) break - except httpx.ReadTimeout: + except (httpx.ReadTimeout, CancelledError): print("Read timeout error. Retrying in 4s...") time.sleep(4) gt_pass_rate = pass_at_k["gt_pass_rate"] From 06437ab9bac7db646f55e4091efeb3bb3941b212 Mon Sep 17 00:00:00 2001 From: Terry Yue Zhuo Date: Thu, 5 Dec 2024 17:16:33 +0800 Subject: [PATCH 215/325] fix(evaluate): remove redundant code --- bigcodebench/evaluate.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 4b00655..f4e01fa 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -395,11 +395,3 @@ def main(): if __name__ == "__main__": main() - -def main(): - from fire import Fire - - Fire(evaluate) - -if __name__ == "__main__": - main() From 8fa95f8bae52bc7c0f7d8e1f92982b561b37ba7c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 5 Dec 2024 20:23:17 +1100 Subject: [PATCH 216/325] feat(evaluate): do calibration by default --- ADVANCED_USAGE.md | 1 + bigcodebench/evaluate.py | 51 ++++++++++++++++++++-------------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 0b2bf7b..fc4ecab 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -69,6 +69,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--local_execute`: Whether to execute the samples locally, default to `False` - `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page. - `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` +- `--calibrated`: Whether to use the calibrated samples, default to `True` - `--save_pass_rate`: Whether to save the pass rate to a file, default to `True` - `--parallel`: The number of parallel processes, default to `-1`, e.g. `--parallel 10` will evaluate 10 samples in parallel - `--min_time_limit`: The minimum time limit for the execution, default to `1`, e.g. `--min_time_limit 10` will evaluate the samples with at least 10 seconds diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index f4e01fa..8f43300 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -122,6 +122,7 @@ def evaluate( remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/", pass_k: str = "1,5,10", save_pass_rate: bool = True, + calibrated: bool = True, parallel: int = -1, min_time_limit: float = 1, max_as_limit: int = 30*1024, @@ -245,7 +246,7 @@ def evaluate( if "solution" in sample else problems[task_id]["complete_prompt"] + sample["completion"] ) - if "sanitized_calibrated" in samples: + if calibrated: solution = problems[task_id]["code_prompt"] + "\n pass\n" + solution remainings.add(sample["_identifier"]) args = ( @@ -266,15 +267,15 @@ def evaluate( assert n_samples == len(remainings), "Missing problems in unfinished" assert len(completion_id) == len(problems), "Missing problems in samples" - def stucking_checker(): - while remainings: - last_size = len(remainings) - time.sleep(240) - if last_size != len(remainings) or len(remainings) == 0: - continue - # Potential stucking - warn("No samples had finished testing in the last 240s") - warn(f"{len(remainings)} samples to be tested: {remainings}") + def stucking_checker(): + while remainings: + last_size = len(remainings) + time.sleep(240) + if last_size != len(remainings) or len(remainings) == 0: + continue + # Potential stucking + warn("No samples had finished testing in the last 240s") + warn(f"{len(remainings)} samples to be tested: {remainings}") threading.Thread(target=stucking_checker).start() @@ -283,20 +284,20 @@ def stucking_checker(): remainings.remove(result["_identifier"]) eval_results[result["task_id"]].append(result) - # sort the results for each problem by completion_id - for task_id, task_results in eval_results.items(): - task_results.sort(key=lambda x: x["completion_id"]) - results["eval"][task_id] = [] - for res in task_results: - stat, details = res["base"] - results["eval"][task_id].append( - { - "task_id": task_id, - "solution": res["solution"], - "status": stat, - "details": details, - } - ) + # sort the results for each problem by completion_id + for task_id, task_results in eval_results.items(): + task_results.sort(key=lambda x: x["completion_id"]) + results["eval"][task_id] = [] + for res in task_results: + stat, details = res["base"] + results["eval"][task_id].append( + { + "task_id": task_id, + "solution": res["solution"], + "status": stat, + "details": details, + } + ) # Calculate pass@k. total = np.array([len(r) for k, r in results["eval"].items() if k in problems]) @@ -319,7 +320,7 @@ def stucking_checker(): pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0] pass_at_k["split"] = split pass_at_k["subset"] = subset - pass_at_k["calibrated"] = "sanitized_calibrated" in samples + pass_at_k["calibrated"] = calibrated pass_at_k["gt_pass_rate"] = gt_pass_rate pass_at_k["failed_tasks"] = failed_tasks From de17cce22e0e66d333baa6f14d73158d9ab054cc Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 5 Dec 2024 20:26:42 +1100 Subject: [PATCH 217/325] fix(evaluate): put the future completion in the executor --- bigcodebench/evaluate.py | 42 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 8f43300..413066b 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -277,27 +277,27 @@ def stucking_checker(): warn("No samples had finished testing in the last 240s") warn(f"{len(remainings)} samples to be tested: {remainings}") - threading.Thread(target=stucking_checker).start() - - for future in tqdm(as_completed(futures), total=n_samples): - result = future.result() - remainings.remove(result["_identifier"]) - eval_results[result["task_id"]].append(result) - - # sort the results for each problem by completion_id - for task_id, task_results in eval_results.items(): - task_results.sort(key=lambda x: x["completion_id"]) - results["eval"][task_id] = [] - for res in task_results: - stat, details = res["base"] - results["eval"][task_id].append( - { - "task_id": task_id, - "solution": res["solution"], - "status": stat, - "details": details, - } - ) + threading.Thread(target=stucking_checker).start() + + for future in tqdm(as_completed(futures), total=n_samples): + result = future.result() + remainings.remove(result["_identifier"]) + eval_results[result["task_id"]].append(result) + + # sort the results for each problem by completion_id + for task_id, task_results in eval_results.items(): + task_results.sort(key=lambda x: x["completion_id"]) + results["eval"][task_id] = [] + for res in task_results: + stat, details = res["base"] + results["eval"][task_id].append( + { + "task_id": task_id, + "solution": res["solution"], + "status": stat, + "details": details, + } + ) # Calculate pass@k. total = np.array([len(r) for k, r in results["eval"].items() if k in problems]) From 25afe4fdacf4f0cfdb53f7389fe4effafcddc195 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 5 Dec 2024 18:20:35 +0800 Subject: [PATCH 218/325] fix: update deps for evaluate --- Docker/Evaluate.Dockerfile | 2 +- Docker/Gradio.Dockerfile | 2 +- release_docker.sh | 5 ----- setup.cfg | 18 ++++++++++++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index 45b7758..69e03e5 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -16,7 +16,7 @@ RUN rm -rf /bigcodebench ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench -RUN cd /bigcodebench && pip install . +RUN cd /bigcodebench && pip install .[evaluate] --no-deps # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index df4018f..2226013 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -18,7 +18,7 @@ RUN rm -rf /bigcodebench ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench -RUN cd /bigcodebench && pip install . +RUN cd /bigcodebench && pip install .[evaluate] --no-deps # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" diff --git a/release_docker.sh b/release_docker.sh index 3b9d104..798a2d6 100755 --- a/release_docker.sh +++ b/release_docker.sh @@ -25,11 +25,6 @@ docker tag bigcodebench/bigcodebench-evaluate:$version bigcodebench/bigcodebench docker push bigcodebench/bigcodebench-evaluate:$version docker push bigcodebench/bigcodebench-evaluate:latest -docker build -f Docker/Generate.Dockerfile . -t bigcodebench/bigcodebench-generate:$version -docker tag bigcodebench/bigcodebench-generate:$version bigcodebench/bigcodebench-generate:latest -docker push bigcodebench/bigcodebench-generate:$version -docker push bigcodebench/bigcodebench-generate:latest - docker build -f Docker/Gradio.Dockerfile . -t bigcodebench/bigcodebench-gradio:$version docker tag bigcodebench/bigcodebench-gradio:$version bigcodebench/bigcodebench-gradio:latest docker push bigcodebench/bigcodebench-gradio:$version diff --git a/setup.cfg b/setup.cfg index 4897f68..73911e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,24 @@ install_requires = mistralai>=0.2.0,<1.0.0 openai>=1.11.1 +[options.extras_require] +# Define optional dependencies (grouped by extras) +evaluate = + appdirs>=1.4.4 + fire>=0.6.0 + multipledispatch>=0.6.0 + pqdm>=0.2.0 + tempdir>=0.7.1 + termcolor>=2.0.0 + tqdm>=4.56.0 + tree_sitter_languages>=1.10.2 + tree-sitter==0.21.3 + wget>=3.2 + datasets + gradio-client + numpy + rich + [options.entry_points] console_scripts = bigcodebench.evaluate = bigcodebench.evaluate:main From 1fa73223ddb33b8efef41fb9dba32dff8706a874 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 5 Dec 2024 18:24:55 +0800 Subject: [PATCH 219/325] fix: change the setup cfg for evaluate --- Docker/Evaluate.Dockerfile | 18 +++++++++++++++++- Docker/Gradio.Dockerfile | 18 +++++++++++++++++- setup.cfg | 18 ------------------ 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index 69e03e5..6bc841b 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -16,7 +16,23 @@ RUN rm -rf /bigcodebench ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench -RUN cd /bigcodebench && pip install .[evaluate] --no-deps +RUN cd /bigcodebench && \ + pip install . --no-deps && \ + pip install \ + appdirs>=1.4.4 \ + fire>=0.6.0 \ + multipledispatch>=0.6.0 \ + pqdm>=0.2.0 \ + tempdir>=0.7.1 \ + termcolor>=2.0.0 \ + tqdm>=4.56.0 \ + tree_sitter_languages>=1.10.2 \ + tree-sitter==0.21.3 \ + wget>=3.2 \ + datasets \ + gradio-client \ + numpy \ + rich # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 2226013..65b86bd 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -18,7 +18,23 @@ RUN rm -rf /bigcodebench ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench -RUN cd /bigcodebench && pip install .[evaluate] --no-deps +RUN cd /bigcodebench && \ + pip install . --no-deps && \ + pip install \ + appdirs>=1.4.4 \ + fire>=0.6.0 \ + multipledispatch>=0.6.0 \ + pqdm>=0.2.0 \ + tempdir>=0.7.1 \ + termcolor>=2.0.0 \ + tqdm>=4.56.0 \ + tree_sitter_languages>=1.10.2 \ + tree-sitter==0.21.3 \ + wget>=3.2 \ + datasets \ + gradio-client \ + numpy \ + rich # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" diff --git a/setup.cfg b/setup.cfg index 73911e7..4897f68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,24 +38,6 @@ install_requires = mistralai>=0.2.0,<1.0.0 openai>=1.11.1 -[options.extras_require] -# Define optional dependencies (grouped by extras) -evaluate = - appdirs>=1.4.4 - fire>=0.6.0 - multipledispatch>=0.6.0 - pqdm>=0.2.0 - tempdir>=0.7.1 - termcolor>=2.0.0 - tqdm>=4.56.0 - tree_sitter_languages>=1.10.2 - tree-sitter==0.21.3 - wget>=3.2 - datasets - gradio-client - numpy - rich - [options.entry_points] console_scripts = bigcodebench.evaluate = bigcodebench.evaluate:main From 36a905d2fd46761f3269b2d48c51b8a572971f73 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 5 Dec 2024 18:33:31 +0800 Subject: [PATCH 220/325] fix: update the tf version for general installation --- Requirements/requirements-eval.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Requirements/requirements-eval.txt b/Requirements/requirements-eval.txt index 1d4a48c..82e1e6b 100644 --- a/Requirements/requirements-eval.txt +++ b/Requirements/requirements-eval.txt @@ -60,7 +60,7 @@ soundfile==0.12.1 statsmodels==0.14.0 statsmodels==0.14.0 sympy==1.12 -tensorflow==2.11.1 +tensorflow==2.11.0 textblob==0.18.0 texttable==1.7.0 Werkzeug==3.0.1 From 3fbcbb66c729134adaa38ce669ee37aada88b46d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 5 Dec 2024 22:27:14 +0800 Subject: [PATCH 221/325] update docker --- Docker/Evaluate.Dockerfile | 4 ++-- Docker/Gradio.Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index 6bc841b..f414340 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -2,7 +2,7 @@ FROM python:3.10-slim # install git, g++ and python3-tk -RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-base +RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-base libgdal-dev # upgrade to latest pip RUN pip install --upgrade pip @@ -37,7 +37,7 @@ RUN cd /bigcodebench && \ # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" -RUN pip install -I --timeout 2000 -r https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0/requirements.txt +RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt WORKDIR /app diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 65b86bd..97fd5c5 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -2,7 +2,7 @@ FROM python:3.10-slim # install git, g++ and python3-tk -RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-base +RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-base libgdal-dev # upgrade to latest pip RUN pip install --upgrade pip @@ -39,7 +39,7 @@ RUN cd /bigcodebench && \ # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" -RUN pip install -I --timeout 2000 -r https://github.com/bigcode-project/bigcodebench-annotation/releases/download/v0.1.0/requirements.txt +RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt RUN apt-get update && \ apt-get install -y \ From b5cfea3b31b00b2719225f5fd846b627bc90d68d Mon Sep 17 00:00:00 2001 From: Terry Yue Zhuo Date: Sat, 7 Dec 2024 21:14:38 +0800 Subject: [PATCH 222/325] fix: update modules in sanitize.py --- bigcodebench/sanitize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigcodebench/sanitize.py b/bigcodebench/sanitize.py index 5a8ab53..2f8ba3b 100644 --- a/bigcodebench/sanitize.py +++ b/bigcodebench/sanitize.py @@ -6,8 +6,8 @@ from pqdm.processes import pqdm from tqdm import tqdm -from tree_sitter import Node -from tree_sitter_languages import get_parser +import tree_sitter_python +from tree_sitter import Language, Node, Parser from bigcodebench.data import ( get_bigcodebench, @@ -111,7 +111,7 @@ def has_return_statement(node: Node) -> bool: def extract_target_code_or_empty(code: str, entrypoint: Optional[str] = None) -> str: code = code_extract(code.strip()) code_bytes = bytes(code, "utf8") - parser = get_parser("python") + parser = Parser(Language(tree_sitter_python.language())) tree = parser.parse(code_bytes) class_names = set() function_names = set() @@ -299,4 +299,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From 3828e6236e1810f3ff31588b23fb39831130ca8c Mon Sep 17 00:00:00 2001 From: Terry Yue Zhuo Date: Sat, 7 Dec 2024 21:15:12 +0800 Subject: [PATCH 223/325] update setup.cfg --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4897f68..6a35ea6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,8 +24,8 @@ install_requires = tempdir>=0.7.1 termcolor>=2.0.0 tqdm>=4.56.0 - tree_sitter_languages>=1.10.2 - tree-sitter==0.21.3 + tree_sitter>=0.22.0 + tree-sitter-python>=0.21.0 wget>=3.2 datasets gradio-client From aa634d5d4a067c979582ba6f28e2bcb588fd4246 Mon Sep 17 00:00:00 2001 From: Terry Yue Zhuo Date: Sat, 7 Dec 2024 21:38:54 +0800 Subject: [PATCH 224/325] fix(evaluate): compute pass_at_k for existing results --- bigcodebench/evaluate.py | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 413066b..ef37068 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -178,6 +178,8 @@ def evaluate( else: + pass_at_k = dict() + pass_k = [int(k) for k in pass_k.split(",")] if parallel < 1: @@ -207,8 +209,6 @@ def evaluate( results = compatible_eval_result(results) else: - pass_at_k = dict() - if check_gt_only: if gt_pass_rate > 0.99: @@ -299,30 +299,30 @@ def stucking_checker(): } ) - # Calculate pass@k. - total = np.array([len(r) for k, r in results["eval"].items() if k in problems]) - base_correct = [] - - for key, res in results["eval"].items(): - if key not in problems: - continue - bc = sum([r["status"] == PASS for r in res]) - base_correct.append(bc) - - base_correct = np.array(base_correct) - - pass_at_k.update({ - f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() - for k in pass_k - if total.min() >= k - }) - - pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0] - pass_at_k["split"] = split - pass_at_k["subset"] = subset - pass_at_k["calibrated"] = calibrated - pass_at_k["gt_pass_rate"] = gt_pass_rate - pass_at_k["failed_tasks"] = failed_tasks + # Calculate pass@k. + total = np.array([len(r) for k, r in results["eval"].items() if k in problems]) + base_correct = [] + + for key, res in results["eval"].items(): + if key not in problems: + continue + bc = sum([r["status"] == PASS for r in res]) + base_correct.append(bc) + + base_correct = np.array(base_correct) + + pass_at_k.update({ + f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() + for k in pass_k + if total.min() >= k + }) + + pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0] + pass_at_k["split"] = split + pass_at_k["subset"] = subset + pass_at_k["calibrated"] = calibrated + pass_at_k["gt_pass_rate"] = gt_pass_rate + pass_at_k["failed_tasks"] = failed_tasks extra = subset.capitalize() split = split.capitalize() From 992fc348b218c6b30cc5f42a46c1ef629172bb42 Mon Sep 17 00:00:00 2001 From: Terry Yue Zhuo Date: Fri, 13 Dec 2024 00:26:40 +0800 Subject: [PATCH 225/325] fix: pass `calibrated` into gradio api --- bigcodebench/evaluate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index ef37068..0eb021f 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -165,6 +165,7 @@ def evaluate( max_as_limit=max_as_limit, max_data_limit=max_data_limit, max_stack_limit=max_stack_limit, + calibrated=calibrated, check_gt_only=check_gt_only, no_gt=no_gt, api_name="/predict" From d92b01259a218c1fdfbded71ebb75d1f4f821450 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 13 Dec 2024 01:55:48 +0800 Subject: [PATCH 226/325] update docker files --- Docker/Evaluate.Dockerfile | 28 +++++++++++++++++++++++----- Docker/Gradio.Dockerfile | 32 +++++++++++++++++++++++++------- release_docker.sh | 23 +++++++++++++++-------- 3 files changed, 63 insertions(+), 20 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index f414340..f3fa7f3 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -2,7 +2,22 @@ FROM python:3.10-slim # install git, g++ and python3-tk -RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-base libgdal-dev +RUN apt-get update && apt-get install -y \ + git \ + g++ \ + python3-tk \ + zip \ + unzip \ + procps \ + r-base \ + libgdal-dev \ + # Add these new dependencies for matplotlib + libfreetype6-dev \ + libpng-dev \ + pkg-config \ + python3-dev \ + python3-matplotlib \ + && rm -rf /var/lib/apt/lists/* # upgrade to latest pip RUN pip install --upgrade pip @@ -16,6 +31,8 @@ RUN rm -rf /bigcodebench ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench +RUN pip install numpy==1.24.3 pyarrow==14.0.1 + RUN cd /bigcodebench && \ pip install . --no-deps && \ pip install \ @@ -29,16 +46,17 @@ RUN cd /bigcodebench && \ tree_sitter_languages>=1.10.2 \ tree-sitter==0.21.3 \ wget>=3.2 \ - datasets \ gradio-client \ - numpy \ rich +RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt + +# Ensure the numpy version is compatible with the datasets version +RUN pip install datasets==2.17.0 + # Pre-install the dataset RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" -RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt - WORKDIR /app RUN chown -R bigcodebenchuser:bigcodebenchuser /app diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 97fd5c5..02d6b29 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -2,8 +2,22 @@ FROM python:3.10-slim # install git, g++ and python3-tk -RUN apt-get update && apt-get install -y git g++ python3-tk zip unzip procps r-base libgdal-dev - +RUN apt-get update && apt-get install -y \ + git \ + g++ \ + python3-tk \ + zip \ + unzip \ + procps \ + r-base \ + libgdal-dev \ + # Add these new dependencies for matplotlib + libfreetype6-dev \ + libpng-dev \ + pkg-config \ + python3-dev \ + python3-matplotlib \ + && rm -rf /var/lib/apt/lists/* # upgrade to latest pip RUN pip install --upgrade pip @@ -18,6 +32,9 @@ RUN rm -rf /bigcodebench ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench + +RUN pip install numpy==1.24.3 pyarrow==14.0.1 + RUN cd /bigcodebench && \ pip install . --no-deps && \ pip install \ @@ -31,16 +48,17 @@ RUN cd /bigcodebench && \ tree_sitter_languages>=1.10.2 \ tree-sitter==0.21.3 \ wget>=3.2 \ - datasets \ gradio-client \ - numpy \ rich -# Pre-install the dataset -RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" - RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt +# Ensure the numpy version is compatible with the datasets version +RUN pip install datasets==2.17.0 + +# Pre-install the dataset +RUN python -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" + RUN apt-get update && \ apt-get install -y \ bash \ diff --git a/release_docker.sh b/release_docker.sh index 798a2d6..896cb4b 100755 --- a/release_docker.sh +++ b/release_docker.sh @@ -20,12 +20,19 @@ fi export PYTHONPATH=$PWD pytest tests -docker build -f Docker/Evaluate.Dockerfile . -t bigcodebench/bigcodebench-evaluate:$version -docker tag bigcodebench/bigcodebench-evaluate:$version bigcodebench/bigcodebench-evaluate:latest -docker push bigcodebench/bigcodebench-evaluate:$version -docker push bigcodebench/bigcodebench-evaluate:latest +docker buildx create --name multiplatform-builder --use || true +docker buildx use multiplatform-builder -docker build -f Docker/Gradio.Dockerfile . -t bigcodebench/bigcodebench-gradio:$version -docker tag bigcodebench/bigcodebench-gradio:$version bigcodebench/bigcodebench-gradio:latest -docker push bigcodebench/bigcodebench-gradio:$version -docker push bigcodebench/bigcodebench-gradio:latest \ No newline at end of file +# Build and push evaluate image +docker buildx build --platform linux/amd64 \ + -f Docker/Evaluate.Dockerfile . \ + -t bigcodebench/bigcodebench-evaluate:$version \ + -t bigcodebench/bigcodebench-evaluate:latest \ + --push + +# Build and push gradio image +docker buildx build --platform linux/amd64 \ + -f Docker/Gradio.Dockerfile . \ + -t bigcodebench/bigcodebench-gradio:$version \ + -t bigcodebench/bigcodebench-gradio:latest \ + --push \ No newline at end of file From 201c2364714001bd145a2e6411b08de14fdb7ba0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 13 Dec 2024 01:56:57 +0800 Subject: [PATCH 227/325] update running example --- run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run.sh b/run.sh index c069e8e..87f80d7 100755 --- a/run.sh +++ b/run.sh @@ -6,6 +6,7 @@ SPLIT=complete SUBSET=hard bigcodebench.evaluate \ + --tp $NUM_GPU \ --model $MODEL \ --split $SPLIT \ --subset $SUBSET \ From 86c05cf4288eda07fc65c8e7cc13c42061f116e1 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 20 Dec 2024 23:22:35 +0800 Subject: [PATCH 228/325] fix: add transformers req --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 6a35ea6..db1fd1d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ install_requires = tree_sitter>=0.22.0 tree-sitter-python>=0.21.0 wget>=3.2 + transformers datasets gradio-client vllm From 90adab8b4452b9c843fbf403a982f6da352339ac Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 20 Dec 2024 23:57:46 +0800 Subject: [PATCH 229/325] update docker --- Docker/Evaluate.Dockerfile | 2 +- Docker/Gradio.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index f3fa7f3..db4c4be 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -55,7 +55,7 @@ RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-p RUN pip install datasets==2.17.0 # Pre-install the dataset -RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" +RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench(subset='full'); get_bigcodebench(subset='hard')" WORKDIR /app diff --git a/Docker/Gradio.Dockerfile b/Docker/Gradio.Dockerfile index 02d6b29..f5110fc 100644 --- a/Docker/Gradio.Dockerfile +++ b/Docker/Gradio.Dockerfile @@ -57,7 +57,7 @@ RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-p RUN pip install datasets==2.17.0 # Pre-install the dataset -RUN python -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench()" +RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench(subset='full'); get_bigcodebench(subset='hard')" RUN apt-get update && \ apt-get install -y \ From 66d9499f208f8e3b6898e5a0a631b4d9f9c470da Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 00:05:58 +0800 Subject: [PATCH 230/325] fix (evaluate): update gt checking --- bigcodebench/evaluate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 0eb021f..066c8e1 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -220,6 +220,8 @@ def evaluate( if len(failed_tasks) > 0: cprint(f"Failed tasks: {failed_tasks}", "red") + return + else: results = { "date": datetime.now().strftime("%Y-%m-%d %H:%M"), From e264513e4629631870e104b4d8a93f5e9b9c4685 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 00:22:20 +0800 Subject: [PATCH 231/325] fix(generate): rm temperature for o1 --- bigcodebench/gen/util/openai_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index a745d8d..c8e7c03 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -15,16 +15,16 @@ def make_request( ) -> ChatCompletion: kwargs["top_p"] = 0.95 kwargs["max_completion_tokens"] = max_tokens + kwargs["temperature"] = temperature if model.startswith("o1-"): # pop top-p and max_completion_tokens kwargs.pop("top_p") kwargs.pop("max_completion_tokens") - + kwargs.pop("temperature") return client.chat.completions.create( model=model, messages=[ {"role": "user", "content": message}, ], - temperature=temperature, n=n, **kwargs ) From e380adffdabc7aa1f27c5d2e540519810ecd284a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 02:08:03 +0800 Subject: [PATCH 232/325] fix(generate): rm max output tokens --- bigcodebench/gen/util/google_request.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bigcodebench/gen/util/google_request.py b/bigcodebench/gen/util/google_request.py index 8e696b4..9e13607 100644 --- a/bigcodebench/gen/util/google_request.py +++ b/bigcodebench/gen/util/google_request.py @@ -11,12 +11,16 @@ def make_request( n: int, max_new_tokens: int = 2048, ) -> genai.types.GenerateContentResponse: + kwargs = {"temperature": temperature, "max_output_tokens": max_new_tokens} + + if "-thinking-" in client.model_name: + kwargs.pop("max_output_tokens") + response = client.generate_content( [{"role": "user", "parts": [message]}], generation_config=genai.types.GenerationConfig( candidate_count=n, - max_output_tokens=max_new_tokens, - temperature=temperature, + **kwargs ), safety_settings=[ {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"}, @@ -44,5 +48,4 @@ def make_auto_request(*args, **kwargs) -> genai.types.GenerateContentResponse: print("Unknown error. Waiting...") print(e) time.sleep(1) - return ret - + return ret \ No newline at end of file From ba18aaf6c92e9c55c9e7f186b9c87027b03872d7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 02:14:47 +0800 Subject: [PATCH 233/325] fix(evaluate): postprocess the concurrent outputs for o1 --- bigcodebench/provider/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 52f8a05..7c35647 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -57,7 +57,7 @@ def _codegen_batch_via_concurrency(self, messages: List[str], num_samples: int) batches = concurrent_call( num_samples, self._codegen_api_batch, messages, num_samples=1 ) - return [b[0] for b in batches] + return [[element for sublist in item for element in sublist] for item in zip(*batches)] def is_direct_completion(self) -> bool: return False \ No newline at end of file From fd3cbc845771d55be2fafbcaab812b084ca823c1 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 02:41:07 +0800 Subject: [PATCH 234/325] fix(generate): use 2nd part for gemini thinking models --- bigcodebench/provider/google.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bigcodebench/provider/google.py b/bigcodebench/provider/google.py index 2194c47..b365c86 100644 --- a/bigcodebench/provider/google.py +++ b/bigcodebench/provider/google.py @@ -43,7 +43,10 @@ def codegen( for candidate in ret.candidates: parts = candidate.content.parts if parts: - outputs.append(parts[0].text) + if "-thinking-" in self.name: + outputs.append(parts[1].text) + else: + outputs.append(parts[0].text) else: print("Empty response!") outputs.append("") From cb30bfcc840a456dc42668ed668044cebd597918 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 04:59:51 +0800 Subject: [PATCH 235/325] update model outputs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index debba10..5b000ec 100755 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ export GOOGLE_API_KEY= ## 💻 LLM-generated Code We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): -* See the attachment of our [v0.2.0.post3](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.0.post3). We include `sanitized_samples_calibrated.zip` for your convenience. +* See the attachment of our [v0.2.1.post7](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.1.post7). We include `sanitized_samples_calibrated.zip` for your convenience. ## 🧑 Advanced Usage From 3314ebe293e3734407692d872cbc170ca98f2955 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 05:09:27 +0800 Subject: [PATCH 236/325] update doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b000ec..9d81126 100755 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ export GOOGLE_API_KEY= ## 💻 LLM-generated Code -We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard): +We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard) on the full set: * See the attachment of our [v0.2.1.post7](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.1.post7). We include `sanitized_samples_calibrated.zip` for your convenience. ## 🧑 Advanced Usage From 24e42d192cafb5e654bf8b8796a28d866206f0fc Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 17:03:39 +0800 Subject: [PATCH 237/325] feat(generate): add reasoning effort for o1 and o3 --- bigcodebench/gen/util/openai_request.py | 5 ++++- bigcodebench/generate.py | 2 ++ bigcodebench/provider/__init__.py | 3 +++ bigcodebench/provider/openai.py | 4 +++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index c8e7c03..1deedf3 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -10,16 +10,19 @@ def make_request( model: str, max_tokens: int = 512, temperature: float = 1, + reasoning_effort: str = "medium", n: int = 1, **kwargs ) -> ChatCompletion: kwargs["top_p"] = 0.95 kwargs["max_completion_tokens"] = max_tokens kwargs["temperature"] = temperature - if model.startswith("o1-"): # pop top-p and max_completion_tokens + if model.startswith("o1-") or model.startswith("o3-"): # pop top-p and max_completion_tokens kwargs.pop("top_p") kwargs.pop("max_completion_tokens") kwargs.pop("temperature") + kwargs["reasoning_effort"] = reasoning_effort + return client.chat.completions.create( model=model, messages=[ diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 757b08c..5acb808 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -132,6 +132,7 @@ def run_codegen( temperature: float = 0.0, max_new_tokens: int = 1280, greedy: bool = False, + reasoning_effort: str = "medium", # o1 and o3 only strip_newlines: bool = False, direct_completion: bool = False, resume: bool = True, @@ -175,6 +176,7 @@ def run_codegen( split=split, temperature=temperature, max_new_tokens=max_new_tokens, + reasoning_effort=reasoning_effort, instruction_prefix=instruction_prefix, response_prefix=response_prefix, base_url=base_url, diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index ef19f4e..af3565c 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -9,6 +9,8 @@ def make_model( dataset: str = "bigcodebench", temperature: float = 0.0, max_new_tokens: int = 1280, + # o1 and o3 only + reasoning_effort: str = "medium", # instruction model only instruction_prefix: str = None, response_prefix: str = None, @@ -73,6 +75,7 @@ def make_model( split=split, temperature=temperature, max_new_tokens=max_new_tokens, + reasoning_effort=reasoning_effort, base_url=base_url, instruction_prefix=instruction_prefix, response_prefix=response_prefix, diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 7c35647..73d9722 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -9,9 +9,10 @@ from bigcodebench.provider.utility import concurrent_call class OpenAIChatDecoder(DecoderBase): - def __init__(self, name: str, base_url=None, **kwargs) -> None: + def __init__(self, name: str, base_url=None, reasoning_effort="medium", **kwargs) -> None: super().__init__(name, **kwargs) self.base_url = base_url + self.reasoning_effort = reasoning_effort def codegen( self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 @@ -45,6 +46,7 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] model=self.name, max_tokens=self.max_new_tokens, temperature=self.temperature, + reasoning_effort=self.reasoning_effort, n=num_samples, ) outputs = [] From e5f27a984ac51bcbe7398ebf275ee498b34ebec0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 21 Dec 2024 17:12:00 +0800 Subject: [PATCH 238/325] fix(generate): update the identifier for o1 and o3 --- bigcodebench/generate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 5acb808..a9ef8d3 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -189,6 +189,8 @@ def run_codegen( ) extra = "-" + subset if subset != "full" else "" + if reasoning_effort and model.startswith("o1-") or model.startswith("o3-"): + model = model + f"--{reasoning_effort}" identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" target_path = os.path.join(root, identifier) From 0532959067216fd4b30b21d72232662307affda9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 26 Dec 2024 03:37:01 +0800 Subject: [PATCH 239/325] feat: add new models --- analysis/utils.py | 168 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 3 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index 4cd9862..974f7d2 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1153,7 +1153,7 @@ # }, "o1-preview-2024-09-12": { "name": "o1-Preview-2024-09-12 (temperature=1)", - "link": "https://o1.ai/o1-preview", + "link": "https://openai.com/index/introducing-openai-o1-preview/", "prompted": True, "moe": False, "size": None, @@ -1162,7 +1162,7 @@ }, "o1-mini-2024-09-12": { "name": "o1-Mini-2024-09-12 (temperature=1)", - "link": "https://o1.ai/o1-preview", + "link": "https://openai.com/index/introducing-openai-o1-preview/", "prompted": True, "moe": False, "size": None, @@ -1331,4 +1331,166 @@ "act_param": None, "open-data": "None", }, -} + "Qwen/Qwen2.5-Coder-32B-Instruct--main": { + "name": "Qwen2.5-Coder-32B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct", + "prompted": True, + "moe": False, + "size": 32, + "act_param": 32, + "open-data": "None", + }, + "infly/OpenCoder-1.5B-Instruct--main": { + "name": "OpenCoder-1.5B-Instruct", + "link": "https://huggingface.co/infly/OpenCoder-1.5B-Instruct", + "prompted": True, + "moe": False, + "size": 1.5, + "act_param": 1.5, + "open-data": "None", + }, + "infly/OpenCoder-8B-Instruct--main": { + "name": "OpenCoder-8B-Instruct", + "link": "https://huggingface.co/infly/OpenCoder-8B-Instruct", + "prompted": True, + "moe": False, + "size": 8, + "act_param": 8, + "open-data": "None", + }, + "microsoft/Phi-3.5-mini-instruct--main": { + "name": "Phi-3.5-Mini-Instruct", + "link": "https://huggingface.co/microsoft/Phi-3.5-mini-instruct", + "prompted": True, + "moe": False, + "size": 3.8, + "act_param": 3.8, + "open-data": "None", + }, + "Nexusflow/Athene-V2-Agent--main": { + "name": "Athene-V2-Agent", + "link": "https://huggingface.co/Nexusflow/Athene-V2-Agent", + "prompted": True, + "moe": False, + "size": 72, + "act_param": 72, + "open-data": "None", + }, + "Nexusflow/Athene-V2-Chat--main": { + "name": "Athene-V2-Chat", + "link": "https://huggingface.co/Nexusflow/Athene-V2-Chat", + "prompted": True, + "moe": False, + "size": 72, + "act_param": 72, + "open-data": "None", + }, + "gemini-exp-1114--main": { + "name": "Gemini-Exp-1114", + "link": "https://deepmind.google/technologies/gemini", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "gpt-4o-2024-11-20--main": { + "name": "GPT-4o-2024-11-20", + "link": "https://openai.com/gpt-4o/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "gemini-exp-1121--main": { + "name": "Gemini-Exp-1121", + "link": "https://deepmind.google/technologies/gemini", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "gemini-exp-1206--main": { + "name": "Gemini-Exp-1206", + "link": "https://deepmind.google/technologies/gemini", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "meta-llama--Llama-3.3-70B-Instruct--main": { + "name": "Llama-3.3-70B-Instruct", + "link": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + }, + "deepseek-ai--DeepSeek-V2.5-1210--main": { + "name": "DeepSeek-V2.5-1210", + "link": "deepseek-ai/DeepSeek-V2.5-1210", + "prompted": True, + "moe": True, + "size": 236, + "act_param": 21, + "open-data": "None", + }, + "gemini-2.0-flash-exp--main": { + "name": "Gemini-2.0-Flash-Exp", + "link": "https://deepmind.google/technologies/gemini", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "gemini-2.0-flash-thinking-exp-1219--main": { + "name": "Gemini-2.0-Flash-Thinking-Exp-1219", + "link": "https://deepmind.google/technologies/gemini", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "o1-2024-12-17--main": { + "name": "o1-2024-12-17 (temperature=1, reasoning=medium)", + "link": "https://openai.com/o1/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "o1-2024-12-17--low--main": { + "name": "o1-2024-12-17 (temperature=1, reasoning=low)", + "link": "https://openai.com/o1/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "o1-2024-12-17--high--main": { + "name": "o1-2024-12-17 (temperature=1, reasoning=high)", + "link": "https://openai.com/o1/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + }, + "deepseek-v3-chat--main": { + "name": "DeepSeek-V3-Chat", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-V3-Chat", + "prompted": True, + "moe": True, + "size": 685, + "act_param": None, + "open-data": "None", + }, +} \ No newline at end of file From 3ac359c457c77c2ae0d03e14968dd94de798e923 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 26 Dec 2024 03:37:55 +0800 Subject: [PATCH 240/325] update the result computation --- analysis/get_results.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index e67fa2a..493ce3d 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -14,6 +14,7 @@ from cuml.linear_model import LogisticRegression import cupy as cp + def update_model_info(model_info): for model, info in model_info.items(): if "https://huggingface.co/" in info["link"]: @@ -56,8 +57,8 @@ def get_results(tids): for model, info in model_info.items(): model = model.replace("/", "--") hf_model = "" - files = glob(f"results/{model}--bigcodebench-*.json") - assert files, f"No files found for results/{model}--bigcodebench-*.json" + files = glob(f"results/{model}--bigcodebench-*_eval_results.json") + assert files, f"No files found for results/{model}--bigcodebench-*_eval_results.json" for file in files: try: _, suffix = os.path.basename(file).split("--bigcodebench-hard-") @@ -86,7 +87,7 @@ def get_results(tids): raise ValueError("Unknown task") mode = "" - if "-sanitized-calibrate" in file: + if "calibrated" in file: mode = "-cal" results[info["name"]][f"pass@1"][f"{task}{mode}"] = round(mean(status)*100,1) @@ -141,17 +142,17 @@ def split_gen(): if "calibrated" in file: if info["prompted"]: if suffix.startswith("complete"): - with open(f"sanitized_calibrated_samples/complete/{model}--bigcodebench-{suffix}", "w") as f: + with open(f"sanitized_calibrated_samples/complete/{model}--bigcodebench*-{suffix}", "w") as f: f.writelines(data) else: - with open(f"sanitized_calibrated_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f: + with open(f"sanitized_calibrated_samples/instruct/{model}--bigcodebench*-{suffix}", "w") as f: f.writelines(data) else: if suffix.startswith("complete"): - with open(f"sanitized_samples/complete/{model}--bigcodebench-{suffix}", "w") as f: + with open(f"sanitized_samples/complete/{model}--bigcodebench*-{suffix}", "w") as f: f.writelines(data) else: - with open(f"sanitized_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f: + with open(f"sanitized_samples/instruct/{model}--bigcodebench*-{suffix}", "w") as f: f.writelines(data) @@ -168,7 +169,7 @@ def read_task_perf(tids, task="complete"): try: try: if info["prompted"]: - files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json") + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized*calibrated_eval_results.json") if files: file = files[0] else: @@ -177,7 +178,7 @@ def read_task_perf(tids, task="complete"): file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0] except: if info["prompted"]:# and not info["direct_complete"]: - files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_hard_eval_results.json") + files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized*calibrated_hard_eval_results.json") if files: file = files[0] else: @@ -187,7 +188,7 @@ def read_task_perf(tids, task="complete"): except: try: if info["prompted"]:# and not info["direct_complete"]: - files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized-calibrated_hard_eval_results.json") + files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized*calibrated_hard_eval_results.json") if files: file = files[0] else: @@ -196,7 +197,7 @@ def read_task_perf(tids, task="complete"): file = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized_hard_eval_results.json")[0] except: if info["prompted"]: - files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized-calibrated_eval_results.json") + files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized*calibrated_eval_results.json") if files: file = files[0] else: @@ -394,7 +395,7 @@ def get_perf_df(data_dict): if __name__ == "__main__": - + split_gen() bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.1") bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.1") bcb_config = { @@ -408,7 +409,7 @@ def get_perf_df(data_dict): instruct_data, instruct_files = read_task_perf(bcb["task_id"], "instruct") complete_df = get_perf_df(complete_data) instruct_df = get_perf_df(instruct_data) - + push_ds(DatasetDict({"complete": Dataset.from_pandas(complete_df), "instruct": Dataset.from_pandas(instruct_df)}), f"bigcode/bigcodebench{suffix}-perf") with open("task2domain.json", "r") as f: From 80c83b692dd955dfd84859bc49e6f45ba035a858 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 26 Dec 2024 15:28:55 +0800 Subject: [PATCH 241/325] add AI2 into the doc --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d81126..4cc8350 100755 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@

## 💥 Impact -BigCodeBench has been used by many LLM teams including: +BigCodeBench has been trusted by many LLM teams including: - Zhipu AI - Alibaba Qwen - DeepSeek @@ -37,6 +37,7 @@ BigCodeBench has been used by many LLM teams including: - Meta AI - Cohere AI - Sakana AI +- Allen Institute for Artificial Intelligence (AI2) ## 📰 News - **[2024-10-06]** We are releasing `bigcodebench==v0.2.0`! From 8cdcdfe6f720d3b245fd1acc5d52c781f4afdf3d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 14 Jan 2025 21:32:24 +0800 Subject: [PATCH 242/325] release bigcodebench data 0.1.3 --- bigcodebench/data/bigcodebench.py | 2 +- tools/fix_v022.py | 59 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tools/fix_v022.py diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 9a3ee9d..26090f1 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -14,7 +14,7 @@ BIGCODEBENCH_OVERRIDE_PATH = os.environ.get("BIGCODEBENCH_OVERRIDE_PATH", None) BIGCODEBENCH_HF = "bigcode/bigcodebench" -BIGCODEBENCH_VERSION = "v0.1.2" +BIGCODEBENCH_VERSION = "v0.1.3" def _ready_bigcodebench_path(subset="full", version="default") -> str: if BIGCODEBENCH_OVERRIDE_PATH: diff --git a/tools/fix_v022.py b/tools/fix_v022.py new file mode 100644 index 0000000..88e1f05 --- /dev/null +++ b/tools/fix_v022.py @@ -0,0 +1,59 @@ +from datasets import load_dataset, Dataset, DatasetDict +from huggingface_hub import HfApi + +import json +import copy + +BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_HARD_HF = "bigcode/bigcodebench-hard" +BIGCODEBENCH_VERSION = "v0.1.2" +BIGCODEBENCH_UPDATE = "bigcode/bcb_update" +BIGCODEBENCH_NEW_VERSION = "v0.1.3" + +def map_ds(sample): + if sample["task_id"] in ["BigCodeBench/1005"]: + for k in sample.keys(): + sample[k] = sample[k].replace( + "https://getsamplefiles.com/download/zip/sample-2.zip", "https://getsamplefiles.com/download/zip/sample-5.zip" + ).replace( + "sample_2", "sample_5" + ).replace( + "Sample 2", "Sample 5" + ) + return sample + +if __name__ == "__main__": + api = HfApi() + ds_dict = load_dataset(BIGCODEBENCH_HF) + hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) + ds = ds_dict[BIGCODEBENCH_VERSION] + hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] + function_id = [1005] + + new_ds = ds.map(map_ds) + new_ds.to_json("BigCodeBench.jsonl") + ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds + ds_dict.push_to_hub(BIGCODEBENCH_HF) + + new_hard_ds = hard_ds.map(map_ds) + new_hard_ds.to_json("BigCodeBench-Hard.jsonl") + hard_ds_dict[BIGCODEBENCH_NEW_VERSION] = new_hard_ds + hard_ds_dict.push_to_hub(BIGCODEBENCH_HARD_HF) + + for i in function_id: + old_sample = ds.select([i]) + new_sample = new_ds.select([i]) + old_sample.to_json("old.jsonl") + new_sample.to_json("new.jsonl") + api.upload_file( + path_or_fileobj="old.jsonl", + path_in_repo=f"{i}/old.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) + api.upload_file( + path_or_fileobj="new.jsonl", + path_in_repo=f"{i}/new.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) From 342aed87bb487c118fd953b52761bd423387f158 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 14 Jan 2025 22:03:16 +0800 Subject: [PATCH 243/325] feat: support selective eval task --- bigcodebench/evaluate.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 066c8e1..6250251 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -119,6 +119,7 @@ def evaluate( samples: Optional[str] = None, no_execute: bool = False, local_execute: bool = False, + selective_evaluate: str = "", remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/", pass_k: str = "1,5,10", save_pass_rate: bool = True, @@ -168,6 +169,7 @@ def evaluate( calibrated=calibrated, check_gt_only=check_gt_only, no_gt=no_gt, + selective_evaluate=selective_evaluate, api_name="/predict" ) break @@ -193,6 +195,14 @@ def evaluate( samples = "__dummy__.jsonl" problems = get_bigcodebench(subset=subset) + + # Add selective evaluation logic + if selective_evaluate: + selected_ids = set(selective_evaluate.split(",")) + problems = {k: v for k, v in problems.items() if k in selected_ids} + if not problems: + raise ValueError(f"None of the provided task IDs {selected_ids} were found in the dataset") + dataset_hash = get_bigcodebench_hash(subset=subset) if not no_gt: @@ -240,10 +250,9 @@ def evaluate( task_id = sample["task_id"] if task_id not in problems: - warn( - f"Task {task_id} is found in the samples but not found in the dataset" - ) + # Skip if task is not in problems (either not in dataset or filtered out by selective_evaluate) continue + solution = ( sample["solution"] if "solution" in sample @@ -267,8 +276,10 @@ def evaluate( completion_id[task_id] += 1 n_samples += 1 + # Modify the assertion to account for selective evaluation assert n_samples == len(remainings), "Missing problems in unfinished" - assert len(completion_id) == len(problems), "Missing problems in samples" + # Only check against problems that weren't filtered out + assert len(completion_id) == len(problems), f"Missing problems in samples. Expected {len(problems)} problems, got {len(completion_id)}" def stucking_checker(): while remainings: From 468eeceb004478f5279617c40003a44bea6e2e3b Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 22 Jan 2025 16:32:38 +0800 Subject: [PATCH 244/325] add e2b support --- bigcodebench/evaluate.py | 31 ++++++++++-- bigcodebench/gen/util/openai_request.py | 2 +- bigcodebench/generate.py | 2 +- bigcodebench/provider/openai.py | 2 +- sandbox-templates/e2b.Dockerfile | 66 +++++++++++++++++++++++++ sandbox-templates/e2b.toml | 16 ++++++ setup.cfg | 1 + 7 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 sandbox-templates/e2b.Dockerfile create mode 100644 sandbox-templates/e2b.toml diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 6250251..19d9a9a 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Tuple, Optional from warnings import warn from gradio_client import Client, handle_file +from e2b import Sandbox import httpx import numpy as np @@ -118,9 +119,10 @@ def evaluate( subset: str, samples: Optional[str] = None, no_execute: bool = False, - local_execute: bool = False, + execution: str = "e2b", # "e2b", "gradio", "local" selective_evaluate: str = "", - remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/", + e2b_endpoint: str = "bigcodebench-evaluator", + gradio_endpoint: str = "https://bigcode-bigcodebench-evaluator.hf.space/", pass_k: str = "1,5,10", save_pass_rate: bool = True, calibrated: bool = True, @@ -152,10 +154,10 @@ def evaluate( assert samples.endswith(".jsonl") result_path = samples.replace(".jsonl", "_eval_results.json") - if not local_execute: + if execution == "gradio": while True: try: - client = Client(remote_execute_api) + client = Client(gradio_endpoint) results, pass_at_k = client.predict( split=split, subset=subset, @@ -178,7 +180,28 @@ def evaluate( time.sleep(4) gt_pass_rate = pass_at_k["gt_pass_rate"] failed_tasks = pass_at_k["failed_tasks"] + + elif execution == "e2b": + sandbox = Sandbox(e2b_endpoint, timeout=60*10) + + # upload file to sandbox + with open(samples, "r") as file: + sandbox.files.write(samples, file) + # run the evaluation + sandbox.commands.run("python3 -m bigcodebench.evaluate \ + --split {} --subset {} --samples {} \ + --pass_k {} --save_pass_rate {} --calibrated {} \ + --parallel {} --min_time_limit {} --max_as_limit {} \ + --max_data_limit {} --max_stack_limit {} --check_gt_only {} --no_gt {} \ + ".format(split, subset, samples, pass_k, save_pass_rate, calibrated, parallel, + min_time_limit, max_as_limit, max_data_limit, max_stack_limit, check_gt_only, no_gt)) + + # download the results + content = sandbox.files.read(result_path) + with open(result_path, "w") as file: + file.write(content) + else: pass_at_k = dict() diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index 1deedf3..f8db3f5 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -17,7 +17,7 @@ def make_request( kwargs["top_p"] = 0.95 kwargs["max_completion_tokens"] = max_tokens kwargs["temperature"] = temperature - if model.startswith("o1-") or model.startswith("o3-"): # pop top-p and max_completion_tokens + if model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): # pop top-p and max_completion_tokens kwargs.pop("top_p") kwargs.pop("max_completion_tokens") kwargs.pop("temperature") diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index a9ef8d3..c7a1983 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -132,7 +132,7 @@ def run_codegen( temperature: float = 0.0, max_new_tokens: int = 1280, greedy: bool = False, - reasoning_effort: str = "medium", # o1 and o3 only + reasoning_effort: str = "medium", strip_newlines: bool = False, direct_completion: bool = False, resume: bool = True, diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 73d9722..12790f6 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -28,7 +28,7 @@ def codegen( tokenizer=None, ) for prompt in prompts] # use concurrency based batching for o1 and deepseek models - if self.name.startswith("o1-") or self.name == "deepseek-chat": + if self.name.startswith("o1-") or self.name.startswith("o3-") or self.name.startswith("deepseek"): return self._codegen_batch_via_concurrency(messages, num_samples) return self._codegen_api_batch(messages, num_samples) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile new file mode 100644 index 0000000..da796d1 --- /dev/null +++ b/sandbox-templates/e2b.Dockerfile @@ -0,0 +1,66 @@ +# Better use newer Python as generated code can use new features +FROM python:3.10-slim + +# install git, g++ and python3-tk +RUN apt-get update && apt-get install -y \ + git \ + g++ \ + python3-tk \ + zip \ + unzip \ + procps \ + r-base \ + libgdal-dev \ + # Add these new dependencies for matplotlib + libfreetype6-dev \ + libpng-dev \ + pkg-config \ + python3-dev \ + python3-matplotlib \ + && rm -rf /var/lib/apt/lists/* + +# upgrade to latest pip +RUN pip install --upgrade pip + +# Add a new user "bigcodebenchuser" +RUN adduser --disabled-password --gecos "" bigcodebenchuser + +RUN rm -rf /bigcodebench + +# Acquire benchmark code to local +ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench + +RUN pip install numpy==1.24.3 pyarrow==14.0.1 + +RUN cd /bigcodebench && \ + pip install . --no-deps && \ + pip install \ + appdirs>=1.4.4 \ + fire>=0.6.0 \ + multipledispatch>=0.6.0 \ + pqdm>=0.2.0 \ + tempdir>=0.7.1 \ + termcolor>=2.0.0 \ + tqdm>=4.56.0 \ + tree_sitter_languages>=1.10.2 \ + tree-sitter==0.21.3 \ + wget>=3.2 \ + gradio-client \ + rich + +RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt + +# Ensure the numpy version is compatible with the datasets version +RUN pip install datasets==2.17.0 + +# Pre-install the dataset +RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench(subset='full'); get_bigcodebench(subset='hard')" + +WORKDIR /app + +RUN chown -R bigcodebenchuser:bigcodebenchuser /app + +RUN chmod -R 777 /app + +USER bigcodebenchuser \ No newline at end of file diff --git a/sandbox-templates/e2b.toml b/sandbox-templates/e2b.toml new file mode 100644 index 0000000..1cdc994 --- /dev/null +++ b/sandbox-templates/e2b.toml @@ -0,0 +1,16 @@ +# This is a config for E2B sandbox template. +# You can use template ID (tbjhnhg5e3bd22i8jqgk) or template name (bigcodebench-evaluator) to create a sandbox: + +# Python SDK +# from e2b import Sandbox, AsyncSandbox +# sandbox = Sandbox("bigcodebench-evaluator") # Sync sandbox +# sandbox = await AsyncSandbox.create("bigcodebench-evaluator") # Async sandbox + +# JS SDK +# import { Sandbox } from 'e2b' +# const sandbox = await Sandbox.create('bigcodebench-evaluator') + +team_id = "f317d0d2-ba02-44c5-8b77-e4a2d7830c7c" +dockerfile = "e2b.Dockerfile" +template_name = "bigcodebench-evaluator" +template_id = "tbjhnhg5e3bd22i8jqgk" diff --git a/setup.cfg b/setup.cfg index db1fd1d..cc20139 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,7 @@ install_requires = google-generativeai>=0.5.4 mistralai>=0.2.0,<1.0.0 openai>=1.11.1 + e2b [options.entry_points] console_scripts = From 62f387c41a4efd4e66b0be66a848359e5e6ea3a1 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 22 Jan 2025 21:16:25 +0800 Subject: [PATCH 245/325] update e2b env setup --- Docker/Evaluate.Dockerfile | 18 ++++++++++++---- bigcodebench/evaluate.py | 34 +++++++++++++++--------------- run.sh | 8 ++++--- sandbox-templates/e2b.Dockerfile | 36 +++++++++++++++++--------------- 4 files changed, 55 insertions(+), 41 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index db4c4be..f6f4af4 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -35,7 +35,8 @@ RUN pip install numpy==1.24.3 pyarrow==14.0.1 RUN cd /bigcodebench && \ pip install . --no-deps && \ - pip install \ + +RUN pip install \ appdirs>=1.4.4 \ fire>=0.6.0 \ multipledispatch>=0.6.0 \ @@ -43,11 +44,20 @@ RUN cd /bigcodebench && \ tempdir>=0.7.1 \ termcolor>=2.0.0 \ tqdm>=4.56.0 \ - tree_sitter_languages>=1.10.2 \ - tree-sitter==0.21.3 \ + tree_sitter>=0.22.0 \ + tree-sitter-python>=0.21.0 \ wget>=3.2 \ + transformers \ + datasets \ gradio-client \ - rich + numpy \ + rich \ + accelerate>=0.30.1 \ + anthropic>=0.26.1 \ + google-generativeai>=0.5.4 + mistralai<1.0.0 \ + openai>=1.11.1 \ + e2b RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 19d9a9a..a98e5bf 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -121,7 +121,7 @@ def evaluate( no_execute: bool = False, execution: str = "e2b", # "e2b", "gradio", "local" selective_evaluate: str = "", - e2b_endpoint: str = "bigcodebench-evaluator", + e2b_endpoint: str = "bigcodebench_evaluator", gradio_endpoint: str = "https://bigcode-bigcodebench-evaluator.hf.space/", pass_k: str = "1,5,10", save_pass_rate: bool = True, @@ -135,7 +135,6 @@ def evaluate( no_gt: bool = False, **model_kwargs, ): - if not samples and model_kwargs: samples = run_codegen( split=split, @@ -182,31 +181,32 @@ def evaluate( failed_tasks = pass_at_k["failed_tasks"] elif execution == "e2b": - sandbox = Sandbox(e2b_endpoint, timeout=60*10) - + sandbox = Sandbox(e2b_endpoint, api_key=os.environ["E2B_API_KEY"], timeout=60*60) + # upload file to sandbox with open(samples, "r") as file: sandbox.files.write(samples, file) # run the evaluation - sandbox.commands.run("python3 -m bigcodebench.evaluate \ - --split {} --subset {} --samples {} \ - --pass_k {} --save_pass_rate {} --calibrated {} \ - --parallel {} --min_time_limit {} --max_as_limit {} \ - --max_data_limit {} --max_stack_limit {} --check_gt_only {} --no_gt {} \ - ".format(split, subset, samples, pass_k, save_pass_rate, calibrated, parallel, - min_time_limit, max_as_limit, max_data_limit, max_stack_limit, check_gt_only, no_gt)) + print(f"Command run in sandbox {e2b_endpoint}") + sandbox.commands.run("bigcodebench.evaluate --execution 'local' " + f"--split {split} --subset {subset} --samples {samples} " + f"--pass_k {pass_k} --save_pass_rate {save_pass_rate} --calibrated {calibrated} " + f"--parallel {parallel} --selective_evaluate {selective_evaluate} --min_time_limit {min_time_limit} " + f"--max_as_limit {max_as_limit} --max_data_limit {max_data_limit} --max_stack_limit {max_stack_limit} " + f"--check_gt_only {check_gt_only} --no_gt {no_gt}", on_stderr=lambda x: print(x), on_stdout=lambda x: print(x), timeout=60*50) - # download the results - content = sandbox.files.read(result_path) - with open(result_path, "w") as file: - file.write(content) + if not check_gt_only: + # download the results + content = sandbox.files.read(result_path) + with open(result_path, "w") as file: + file.write(content) else: pass_at_k = dict() - pass_k = [int(k) for k in pass_k.split(",")] + passk = [int(k) for k in pass_k.split(",")] if parallel < 1: n_workers = max(1, multiprocessing.cpu_count() // 2) @@ -350,7 +350,7 @@ def stucking_checker(): pass_at_k.update({ f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() - for k in pass_k + for k in passk if total.min() >= k }) diff --git a/run.sh b/run.sh index 87f80d7..518a3ae 100755 --- a/run.sh +++ b/run.sh @@ -3,11 +3,13 @@ MODEL=meta-llama/Meta-Llama-3.1-8B-Instruct BACKEND=vllm NUM_GPU=2 SPLIT=complete -SUBSET=hard +SUBSET=full +export E2B_API_KEY="e2b_0a231fa3b0a2b01690ab6c66a23b55c0979ce4ee" bigcodebench.evaluate \ - --tp $NUM_GPU \ + --samples bcb_results/gemini-2.0-flash-exp--main--bigcodebench-instruct--google-0-1-sanitized_calibrated.jsonl \ --model $MODEL \ --split $SPLIT \ --subset $SUBSET \ - --backend $BACKEND \ No newline at end of file + --backend $BACKEND \ + --check_gt_only \ No newline at end of file diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index da796d1..c6ba2ca 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -34,33 +34,35 @@ RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 RUN cd /bigcodebench && \ - pip install . --no-deps && \ - pip install \ - appdirs>=1.4.4 \ - fire>=0.6.0 \ - multipledispatch>=0.6.0 \ - pqdm>=0.2.0 \ - tempdir>=0.7.1 \ - termcolor>=2.0.0 \ - tqdm>=4.56.0 \ - tree_sitter_languages>=1.10.2 \ - tree-sitter==0.21.3 \ - wget>=3.2 \ + pip install . --no-deps + +RUN pip install --timeout 2000 \ + appdirs \ + fire \ + multipledispatch \ + pqdm \ + tempdir \ + termcolor \ + tqdm \ + transformers \ + tree_sitter \ + tree-sitter-python \ + wget \ + datasets \ gradio-client \ - rich + numpy \ + rich \ + e2b RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt # Ensure the numpy version is compatible with the datasets version RUN pip install datasets==2.17.0 -# Pre-install the dataset -RUN python3 -c "from bigcodebench.data import get_bigcodebench; get_bigcodebench(subset='full'); get_bigcodebench(subset='hard')" - WORKDIR /app RUN chown -R bigcodebenchuser:bigcodebenchuser /app -RUN chmod -R 777 /app +RUN chmod -R 777 /app && rm -rf /root/.cache/pip USER bigcodebenchuser \ No newline at end of file From 092c5a3ed582fb58da4ffabb211189c62b2b8683 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 22 Jan 2025 22:28:34 +0800 Subject: [PATCH 246/325] update e2b toml --- sandbox-templates/e2b.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sandbox-templates/e2b.toml b/sandbox-templates/e2b.toml index 1cdc994..ad6a1f7 100644 --- a/sandbox-templates/e2b.toml +++ b/sandbox-templates/e2b.toml @@ -1,16 +1,16 @@ # This is a config for E2B sandbox template. -# You can use template ID (tbjhnhg5e3bd22i8jqgk) or template name (bigcodebench-evaluator) to create a sandbox: +# You can use template ID (xs3c9i0hy53751xam77h) or template name (bigcodebench_evaluator) to create a sandbox: # Python SDK # from e2b import Sandbox, AsyncSandbox -# sandbox = Sandbox("bigcodebench-evaluator") # Sync sandbox -# sandbox = await AsyncSandbox.create("bigcodebench-evaluator") # Async sandbox +# sandbox = Sandbox("bigcodebench_evaluator") # Sync sandbox +# sandbox = await AsyncSandbox.create("bigcodebench_evaluator") # Async sandbox # JS SDK # import { Sandbox } from 'e2b' -# const sandbox = await Sandbox.create('bigcodebench-evaluator') +# const sandbox = await Sandbox.create('bigcodebench_evaluator') team_id = "f317d0d2-ba02-44c5-8b77-e4a2d7830c7c" dockerfile = "e2b.Dockerfile" -template_name = "bigcodebench-evaluator" -template_id = "tbjhnhg5e3bd22i8jqgk" +template_name = "bigcodebench_evaluator" +template_id = "xs3c9i0hy53751xam77h" From 57a4a3cbc417b0ba9aa58f0e307bb803f90899d2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 23 Jan 2025 01:21:19 +0800 Subject: [PATCH 247/325] add r1 reasoning effort --- bigcodebench/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index c7a1983..dfcbc7c 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -189,7 +189,7 @@ def run_codegen( ) extra = "-" + subset if subset != "full" else "" - if reasoning_effort and model.startswith("o1-") or model.startswith("o3-"): + if reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): model = model + f"--{reasoning_effort}" identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" From d0cacd095f4cf3aed491b9848276f37bd367cb37 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 23 Jan 2025 01:56:20 +0800 Subject: [PATCH 248/325] fix docker --- Docker/Evaluate.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index f6f4af4..3a6e780 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -54,7 +54,7 @@ RUN pip install \ rich \ accelerate>=0.30.1 \ anthropic>=0.26.1 \ - google-generativeai>=0.5.4 + google-generativeai>=0.5.4 \ mistralai<1.0.0 \ openai>=1.11.1 \ e2b From 65305f1cf5cc945ba8335898c0822b2fea1f1569 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 23 Jan 2025 02:00:55 +0800 Subject: [PATCH 249/325] fix docker --- Docker/Evaluate.Dockerfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index 3a6e780..e95cbf6 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -37,26 +37,26 @@ RUN cd /bigcodebench && \ pip install . --no-deps && \ RUN pip install \ - appdirs>=1.4.4 \ - fire>=0.6.0 \ - multipledispatch>=0.6.0 \ - pqdm>=0.2.0 \ - tempdir>=0.7.1 \ - termcolor>=2.0.0 \ - tqdm>=4.56.0 \ - tree_sitter>=0.22.0 \ - tree-sitter-python>=0.21.0 \ - wget>=3.2 \ + appdirs \ + fire \ + multipledispatch \ + pqdm \ + tempdir \ + termcolor \ + tqdm \ + tree_sitter \ + tree-sitter-python \ + wget \ transformers \ datasets \ gradio-client \ numpy \ rich \ - accelerate>=0.30.1 \ - anthropic>=0.26.1 \ - google-generativeai>=0.5.4 \ - mistralai<1.0.0 \ - openai>=1.11.1 \ + accelerate \ + anthropic \ + google-generativeai \ + mistralai \ + openai \ e2b RUN pip install -I --timeout 2000 -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/refs/heads/main/Requirements/requirements-eval.txt From 11c0080924a0d73cf075126b5cd4134bbe949def Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 23 Jan 2025 02:02:30 +0800 Subject: [PATCH 250/325] fix docker --- Docker/Evaluate.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index e95cbf6..90e7f40 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -34,7 +34,7 @@ RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 RUN cd /bigcodebench && \ - pip install . --no-deps && \ + pip install . --no-deps RUN pip install \ appdirs \ From 6ffa085a344e7f0b49080b2c3190a41b2b3698e4 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 23 Jan 2025 02:27:01 +0800 Subject: [PATCH 251/325] update doc --- ADVANCED_USAGE.md | 5 ++++- README.md | 18 ++++++++++++++++-- bigcodebench/evaluate.py | 2 +- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index fc4ecab..14c38f3 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -56,6 +56,8 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--resume`: Whether to resume the evaluation, default to `True`, set to `False` to re-run the evaluation - `--id_range`: The range of the tasks to evaluate, default to `None`, e.g. `--id_range 10-20` will evaluate the tasks from 10 to 20 - `--backend`: The backend to use, default to `vllm` +- `--execution`: The execution backend to use, default to `gradio`. You can choose from `e2b`, `gradio`, `local`. +- `--reasoning_effort`: The reasoning effort to use, default to `medium`. You can choose from `easy`, `medium`, `hard` for `o1`, `o3` and `deepseek-reasoner`(soon) models. - `--base_url`: The base URL of the backend for OpenAI-compatible APIs, default to `None` - `--instruction_prefix`: The instruction prefix for the Anthropic backend, default to `None` - `--response_prefix`: The response prefix for the Anthropic backend, default to `None` @@ -67,7 +69,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--samples`: The path to the generated samples file, default to `None` - `--no_execute`: Whether to not execute the samples, default to `False` - `--local_execute`: Whether to execute the samples locally, default to `False` -- `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page. +- `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page - `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` - `--calibrated`: Whether to use the calibrated samples, default to `True` - `--save_pass_rate`: Whether to save the pass rate to a file, default to `True` @@ -76,6 +78,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--max_as_limit`: The maximum address space limit for the execution, default to `30*1024` (30 GB), e.g. `--max_as_limit 20*1024` will evaluate the samples with at most 20 GB - `--max_data_limit`: The maximum data segment limit for the execution, default to `30*1024` (30 GB), e.g. `--max_data_limit 20*1024` will evaluate the samples with at most 20 GB - `--max_stack_limit`: The maximum stack limit for the execution, default to `10`, e.g. `--max_stack_limit 20` will evaluate the samples with at most 20 MB +- `--selective_evaluate`: The subset of the dataset to evaluate, default to `""`. You can pass the index of the tasks to evaluate, e.g. `--selective_evaluate 1,2,3` will evaluate the BigCodeBench/1, BigCodeBench/2 and BigCodeBench/3 - `--check_gt_only`: Whether to only check the ground truths, default to `False` - `--no_gt`: Whether to not check the ground truths, default to `False` diff --git a/README.md b/README.md index 4cc8350..f015c15 100755 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ -

@@ -40,6 +39,7 @@ BigCodeBench has been trusted by many LLM teams including: - Allen Institute for Artificial Intelligence (AI2) ## 📰 News +- **[2025-01-22]** We are releasing `bigcodebench==v0.2.2.dev2`, with 163 models evaluated! - **[2024-10-06]** We are releasing `bigcodebench==v0.2.0`! - **[2024-10-05]** We create a public code execution API on the [Hugging Face space](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator). - **[2024-10-01]** We have evaluated 139 models on BigCodeBench-Hard so far. Take a look at the [leaderboard](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard)! @@ -111,11 +111,13 @@ We use the greedy decoding as an example to show how to evaluate the generated c > [!Note] > -> Remotely executing on `BigCodeBench-Full` typically takes 6-7 minutes, and on `BigCodeBench-Hard` typically takes 4-5 minutes. +> `gradio` backend on `BigCodeBench-Full` typically takes 6-7 minutes, and on `BigCodeBench-Hard` typically takes 4-5 minutes. +> `e2b` backend with default machine on `BigCodeBench-Full` typically takes 25-30 minutes, and on `BigCodeBench-Hard` typically takes 15-20 minutes. ```bash bigcodebench.evaluate \ --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --execution [e2b|gradio|local] \ --split [complete|instruct] \ --subset [full|hard] \ --backend [vllm|openai|anthropic|google|mistral|hf] @@ -126,6 +128,12 @@ bigcodebench.evaluate \ - The evaluation results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_eval_results.json`. - The pass@k results will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples]-sanitized_calibrated_pass_at_k.json`. +> [!Note] +> +> The `gradio` backend is hosted on the [Hugging Face space](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) by default. +> The default space can be sometimes slow, so we recommend you to use the `e2b` backend for faster evaluation. +> Otherwise, you can also use the `e2b` sandbox for evaluation, which is also pretty slow on the default machine. + > [!Note] > > BigCodeBench uses different prompts for base and chat models. @@ -136,6 +144,12 @@ bigcodebench.evaluate \ > please add `--direct_completion` to avoid being evaluated > in a chat mode. +To use E2B, you need to set up an account and get an API key from [E2B](https://e2b.dev/). + +```bash +export E2B_API_KEY= +``` + Access OpenAI APIs from [OpenAI Console](https://platform.openai.com/) ```bash export OPENAI_API_KEY= diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index a98e5bf..e242ce4 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -119,7 +119,7 @@ def evaluate( subset: str, samples: Optional[str] = None, no_execute: bool = False, - execution: str = "e2b", # "e2b", "gradio", "local" + execution: str = "gradio", # "e2b", "gradio", "local" selective_evaluate: str = "", e2b_endpoint: str = "bigcodebench_evaluator", gradio_endpoint: str = "https://bigcode-bigcodebench-evaluator.hf.space/", From e62c55e85703835128189d09b10ae64cd53bd153 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 23 Jan 2025 23:37:51 +0800 Subject: [PATCH 252/325] update google thinking model api --- bigcodebench/provider/google.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bigcodebench/provider/google.py b/bigcodebench/provider/google.py index b365c86..2194c47 100644 --- a/bigcodebench/provider/google.py +++ b/bigcodebench/provider/google.py @@ -43,10 +43,7 @@ def codegen( for candidate in ret.candidates: parts = candidate.content.parts if parts: - if "-thinking-" in self.name: - outputs.append(parts[1].text) - else: - outputs.append(parts[0].text) + outputs.append(parts[0].text) else: print("Empty response!") outputs.append("") From fdbc50fc3df16e6105a14e4664df8730b5886cdc Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 23 Jan 2025 23:40:36 +0800 Subject: [PATCH 253/325] update doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f015c15..2c0fc55 100755 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ bigcodebench.evaluate \ > [!Note] > > The `gradio` backend is hosted on the [Hugging Face space](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) by default. -> The default space can be sometimes slow, so we recommend you to use the `e2b` backend for faster evaluation. +> The default space can be sometimes slow, so we recommend you to use the `gradio` backend with a cloned [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) endpoint for faster evaluation. > Otherwise, you can also use the `e2b` sandbox for evaluation, which is also pretty slow on the default machine. > [!Note] From a4f300ab016c1591040c03483da336d8c1154171 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 20:15:50 +0800 Subject: [PATCH 254/325] fix: remove sample file in example --- run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/run.sh b/run.sh index 518a3ae..6242abd 100755 --- a/run.sh +++ b/run.sh @@ -7,7 +7,6 @@ SUBSET=full export E2B_API_KEY="e2b_0a231fa3b0a2b01690ab6c66a23b55c0979ce4ee" bigcodebench.evaluate \ - --samples bcb_results/gemini-2.0-flash-exp--main--bigcodebench-instruct--google-0-1-sanitized_calibrated.jsonl \ --model $MODEL \ --split $SPLIT \ --subset $SUBSET \ From 746a19907634c8f7f2a77559dffe0701744ebcd5 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 20:23:32 +0800 Subject: [PATCH 255/325] feat: make prefill optional --- bigcodebench/generate.py | 11 ++++------- bigcodebench/provider/__init__.py | 3 +++ bigcodebench/provider/base.py | 2 ++ bigcodebench/provider/utility.py | 23 ++++++++++++++++------- bigcodebench/provider/vllm.py | 1 + 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index dfcbc7c..5a6d9db 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -140,8 +140,9 @@ def run_codegen( backend: str = "vllm", base_url: str = None, tp: int = 1, - instruction_prefix: str = None, - response_prefix: str = None, + instruction_prefix: str = "Please provide a self-contained Python script that solves the following problem in a markdown code block:", + response_prefix: str ="Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:", + prefill: bool = True, revision: str = "main", trust_remote_code: bool = False, tokenizer_name: str = None, @@ -163,11 +164,6 @@ def run_codegen( # Make project dir os.makedirs(root, exist_ok=True) - if instruction_prefix is None: - instruction_prefix = "Please provide a self-contained Python script that solves the following problem in a markdown code block:" - if response_prefix is None: - response_prefix = "Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:" - # Make dir for codes generated by each model model_runner = make_model( model=model, @@ -179,6 +175,7 @@ def run_codegen( reasoning_effort=reasoning_effort, instruction_prefix=instruction_prefix, response_prefix=response_prefix, + prefill=prefill, base_url=base_url, tp=tp, revision=revision, diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index af3565c..dbadfd4 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -14,6 +14,7 @@ def make_model( # instruction model only instruction_prefix: str = None, response_prefix: str = None, + prefill: bool = True, # vllm and hf only revision: str = "main", # vllm only @@ -42,6 +43,7 @@ def make_model( tp=tp, instruction_prefix=instruction_prefix, response_prefix=response_prefix, + prefill=prefill, trust_remote_code=trust_remote_code, tokenizer_name=tokenizer_name, tokenizer_legacy=tokenizer_legacy, @@ -60,6 +62,7 @@ def make_model( direct_completion=direct_completion, instruction_prefix=instruction_prefix, response_prefix=response_prefix, + prefill=prefill, attn_implementation=attn_implementation, trust_remote_code=trust_remote_code, tokenizer_name=tokenizer_name, diff --git a/bigcodebench/provider/base.py b/bigcodebench/provider/base.py index 5a24b59..2bfec2a 100644 --- a/bigcodebench/provider/base.py +++ b/bigcodebench/provider/base.py @@ -20,6 +20,7 @@ def __init__( tokenizer_legacy: bool = False, instruction_prefix: str = None, response_prefix: str = None, + prefill: bool = True, ) -> None: print("Initializing a decoder model: {} ...".format(name)) self.name = name @@ -37,6 +38,7 @@ def __init__( self.tokenizer_legacy = tokenizer_legacy self.instruction_prefix = instruction_prefix self.response_prefix = response_prefix + self.prefill = prefill @abstractmethod def codegen( diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py index bb27539..d363533 100644 --- a/bigcodebench/provider/utility.py +++ b/bigcodebench/provider/utility.py @@ -28,6 +28,7 @@ def make_raw_chat_prompt( split: str, instruction_prefix: str, response_prefix: str, + prefill: bool, tokenizer: AutoTokenizer, direct_completion: bool = False, ) -> str: @@ -58,13 +59,21 @@ def make_raw_chat_prompt( ``` """ if tokenizer: - task_prompt = tokenizer.apply_chat_template( - [ - {"role": "user", "content": task_prompt}, - {"role": "assistant", "content": response}, - ], - tokenize=False, - ).split(_MAGIC_SPLITTER_)[0] + if prefill: + task_prompt = tokenizer.apply_chat_template( + [ + {"role": "user", "content": task_prompt}, + {"role": "assistant", "content": response}, + ], + tokenize=False, + ).split(_MAGIC_SPLITTER_)[0] + else: + task_prompt = tokenizer.apply_chat_template( + [ + {"role": "user", "content": task_prompt}, + ], + tokenize=False, + ).split(_MAGIC_SPLITTER_)[0] return task_prompt diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 171a41c..5ce67ab 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -47,6 +47,7 @@ def codegen( split=self.split, instruction_prefix=self.instruction_prefix, response_prefix=self.response_prefix, + prefill=self.prefill, tokenizer=self.tokenizer, direct_completion=self.direct_completion, ) From 2ff547d5dcb5da36131caa918ada08c3285c7a21 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 20:38:03 +0800 Subject: [PATCH 256/325] fix: change prefill to no_prefill --- bigcodebench/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 5a6d9db..929e72d 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -142,7 +142,7 @@ def run_codegen( tp: int = 1, instruction_prefix: str = "Please provide a self-contained Python script that solves the following problem in a markdown code block:", response_prefix: str ="Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:", - prefill: bool = True, + no_prefill: bool = False, revision: str = "main", trust_remote_code: bool = False, tokenizer_name: str = None, @@ -175,7 +175,7 @@ def run_codegen( reasoning_effort=reasoning_effort, instruction_prefix=instruction_prefix, response_prefix=response_prefix, - prefill=prefill, + prefill=not no_prefill, base_url=base_url, tp=tp, revision=revision, From 5e17316551bf5b1b8bab63c1e0577b0b37c4b9a9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 20:45:03 +0800 Subject: [PATCH 257/325] feat: add no_prefill in file if being activated --- bigcodebench/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 929e72d..cbca10b 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -188,7 +188,7 @@ def run_codegen( extra = "-" + subset if subset != "full" else "" if reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): model = model + f"--{reasoning_effort}" - identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" + identifier = model.replace("/", "--") + "--no_prefill" if no_prefill else "" + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" target_path = os.path.join(root, identifier) From 0f30a64c26dc0516d90a5b887cdc0b8b8e5c2cd7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 21:29:12 +0800 Subject: [PATCH 258/325] fix: change arg --- bigcodebench/generate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index cbca10b..866f59d 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -142,7 +142,7 @@ def run_codegen( tp: int = 1, instruction_prefix: str = "Please provide a self-contained Python script that solves the following problem in a markdown code block:", response_prefix: str ="Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:", - no_prefill: bool = False, + skip_prefill: bool = False, revision: str = "main", trust_remote_code: bool = False, tokenizer_name: str = None, @@ -175,7 +175,7 @@ def run_codegen( reasoning_effort=reasoning_effort, instruction_prefix=instruction_prefix, response_prefix=response_prefix, - prefill=not no_prefill, + prefill=not skip_prefill, base_url=base_url, tp=tp, revision=revision, @@ -188,7 +188,7 @@ def run_codegen( extra = "-" + subset if subset != "full" else "" if reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): model = model + f"--{reasoning_effort}" - identifier = model.replace("/", "--") + "--no_prefill" if no_prefill else "" + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" + identifier = model.replace("/", "--") + "--skip_prefill" if skip_prefill else "" + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" target_path = os.path.join(root, identifier) From 1e12249fd952e94a203d7b9b1da68e805b275b24 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 21:40:58 +0800 Subject: [PATCH 259/325] fix gen file name --- bigcodebench/generate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 866f59d..bcf1463 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -188,7 +188,11 @@ def run_codegen( extra = "-" + subset if subset != "full" else "" if reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): model = model + f"--{reasoning_effort}" - identifier = model.replace("/", "--") + "--skip_prefill" if skip_prefill else "" + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" + + if skip_prefill: + identifier = model.replace("/", "--") + "--skip_prefill" + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" + else: + identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" target_path = os.path.join(root, identifier) From 832035eb0dac96f09d5e81519223f082e485d67e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 22:44:38 +0800 Subject: [PATCH 260/325] update eval model --- analysis/get_results.py | 126 +------------ analysis/utils.py | 400 ++++++++++++++++++++++++++++++++-------- 2 files changed, 333 insertions(+), 193 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index 493ce3d..fc5aa17 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -11,9 +11,6 @@ import math from datasets import Dataset, DatasetDict, load_dataset from transformers import AutoTokenizer -from cuml.linear_model import LogisticRegression -import cupy as cp - def update_model_info(model_info): for model, info in model_info.items(): @@ -142,17 +139,17 @@ def split_gen(): if "calibrated" in file: if info["prompted"]: if suffix.startswith("complete"): - with open(f"sanitized_calibrated_samples/complete/{model}--bigcodebench*-{suffix}", "w") as f: + with open(f"sanitized_calibrated_samples/complete/{model}--bigcodebench-{suffix}", "w") as f: f.writelines(data) else: - with open(f"sanitized_calibrated_samples/instruct/{model}--bigcodebench*-{suffix}", "w") as f: + with open(f"sanitized_calibrated_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f: f.writelines(data) else: if suffix.startswith("complete"): - with open(f"sanitized_samples/complete/{model}--bigcodebench*-{suffix}", "w") as f: + with open(f"sanitized_samples/complete/{model}--bigcodebench-{suffix}", "w") as f: f.writelines(data) else: - with open(f"sanitized_samples/instruct/{model}--bigcodebench*-{suffix}", "w") as f: + with open(f"sanitized_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f: f.writelines(data) @@ -221,95 +218,6 @@ def read_task_perf(tids, task="complete"): return model_results, result_files -def get_winner_df(data_dict, tids, task, task_level=True, no_tie=True): - winner_dict = {"task_id": [], "model_a": [], "model_b": [], "winner": []} - if not task_level: - file = f"{task}_winner_df.csv" - else: - file = f"{task}_winner_task_df.csv" - - if task_level: - for task_id in tqdm(tids): - # pair without repetition (a, b) and (b, a) are the same - for model_a, model_b in itertools.combinations(data_dict.keys(), 2): - solve_rate_a = data_dict[model_a][task_id] - solve_rate_b = data_dict[model_b][task_id] - - if solve_rate_a > solve_rate_b: - winner_dict["winner"].append("model_a") - elif solve_rate_a < solve_rate_b: - winner_dict["winner"].append("model_b") - else: - if no_tie: - continue - winner_dict["winner"].append("tie") - - winner_dict["task_id"].append(task_id) - winner_dict["model_a"].append(model_a) - winner_dict["model_b"].append(model_b) - else: - data_dict = {model: np.mean(list(task_perf.values())) for model, task_perf in data_dict.items()} - for model_a, model_b in itertools.combinations(data_dict.keys(), 2): - solve_rate_a = data_dict[model_a] - solve_rate_b = data_dict[model_b] - - if solve_rate_a > solve_rate_b: - winner_dict["winner"].append("model_a") - elif solve_rate_a < solve_rate_b: - winner_dict["winner"].append("model_b") - else: - if no_tie: - continue - winner_dict["winner"].append("tie") - winner_dict["task_id"].append(task) - winner_dict["model_a"].append(model_a) - winner_dict["model_b"].append(model_b) - - df = pd.DataFrame(winner_dict) - df.to_csv(file, index=False) - return df - - -def get_bootstrap_result(battles, func_compute_elo, num_round): - rows = [] - for i in tqdm(range(num_round), desc="bootstrap"): - rows.append(func_compute_elo(battles.sample(frac=1.0, replace=True))) - df = pd.DataFrame(rows) - return df[df.median().sort_values(ascending=False).index] - - -def get_elo_mle(df, SCALE=400, BASE=10, INIT_RATING=1000): - - - models = pd.concat([df["model_a"], df["model_b"]]).unique() - models = pd.Series(np.arange(len(models)), index=models) - p = len(models.index) - n = df.shape[0] - - X = cp.zeros([n, p]) - X[cp.arange(n), models[df["model_a"]]] = +math.log(BASE) - X[cp.arange(n), models[df["model_b"]]] = -math.log(BASE) - - Y = cp.zeros(n) - Y[df["winner"] == "model_a"] = 1.0 - - lr = LogisticRegression(fit_intercept=False) - lr.fit(X, Y) - - elo_scores = SCALE * lr.coef_[0] + INIT_RATING - - return pd.Series(cp.asnumpy(elo_scores), index=models.index).sort_values(ascending=False) - - -def update_elo_rating(results, elo_dict): - for model, info in model_info.items(): - if info["name"] not in elo_dict: - results[info["name"]]["elo_mle"] = None - else: - results[info["name"]]["elo_mle"] = elo_dict[info["name"]] - return results - - def get_domain_perf(data_dict, task2domain): domain_perfs = { "Model": [], @@ -347,7 +255,7 @@ def get_solve_rate(data_dict, task="complete"): def get_hf_ds(results): hf_dataset = {"model": [], "link": [], "moe": [], "size": [], "act_param": [], "type": [], #"lazy": [],# "direct_complete": [], - "complete": [], "instruct": [], "elo_mle": []} + "complete": [], "instruct": []} for model, result in results.items(): hf_dataset["model"].append(model) @@ -360,7 +268,6 @@ def get_hf_ds(results): hf_dataset["complete"].append(result["pass@1"]["complete"]) hf_dataset["instruct"].append(result["pass@1"]["instruct"]) # hf_dataset["direct_complete"].append(result["direct_complete"]) - hf_dataset["elo_mle"].append(result["elo_mle"]) return Dataset.from_dict(hf_dataset) @@ -395,7 +302,7 @@ def get_perf_df(data_dict): if __name__ == "__main__": - split_gen() + # split_gen() bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.1") bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.1") bcb_config = { @@ -429,28 +336,7 @@ def get_perf_df(data_dict): instruct_solve_rate = get_solve_rate(instruct_data, task="instruct") solve_rate_ds = DatasetDict({"complete": complete_solve_rate, "instruct": instruct_solve_rate}) push_ds(solve_rate_ds, f"bigcode/bigcodebench{suffix}-solve-rate") - - elo_config = { - "task_no_tie": (True, True), - "benchmark_tie": (False, False), - } - elo_ds = dict() - for config, (task_level, no_tie) in elo_config.items(): - filter_complete_data = {model: task_perf for model, task_perf in complete_data.items() if model in instruct_data} - complete_battles = get_winner_df(filter_complete_data, bcb["task_id"], "complete", task_level=task_level, no_tie=no_tie) - instruct_battles = get_winner_df(instruct_data, bcb["task_id"], "instruct", task_level=task_level, no_tie=no_tie) - battles = pd.concat([complete_battles, instruct_battles]) - elo_mle_bootstrap = get_bootstrap_result(battles, get_elo_mle, 500) - bootstrap_lu_median = elo_mle_bootstrap.median().reset_index().set_axis(["model", "Elo rating"], axis=1) - bootstrap_lu_median["Elo rating"] = (bootstrap_lu_median["Elo rating"] + 0.5).astype(int) - bootstrap_lu_median_dict = bootstrap_lu_median.set_index("model")["Elo rating"].to_dict() - if config == "task_no_tie": - task_elo = bootstrap_lu_median_dict - elo = get_bootstrap_scores(elo_mle_bootstrap) - elo_ds[config] = elo - push_ds(DatasetDict(elo_ds), f"bigcode/bigcodebench{suffix}-elo") - results = update_elo_rating(results, task_elo) with open(f"results{suffix}.json", "w") as f: json.dump(results, f, indent=4) ds = get_hf_ds(results) diff --git a/analysis/utils.py b/analysis/utils.py index 974f7d2..c0f2a64 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -7,7 +7,8 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", - }, + "reasoning": False, + }, "bigcode/starcoder2-15b-instruct-v0.1": { "name": "StarCoder2-15B-Instruct-v0.1", "link": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1", @@ -16,7 +17,8 @@ "size": 15, "act_param": 15, "open-data": "Full", - }, + "reasoning": False, + }, "bigcode/starcoder2-3b": { "name": "StarCoder2-3B", "link": "https://huggingface.co/bigcode/starcoder2-3b", @@ -25,7 +27,8 @@ "size": 3, "act_param": 3, "open-data": "Full", - }, + "reasoning": False, + }, "bigcode/starcoder2-7b": { "name": "StarCoder2-7B", "link": "https://huggingface.co/bigcode/starcoder2-7b", @@ -34,7 +37,8 @@ "size": 7, "act_param": 7, "open-data": "Full", - }, + "reasoning": False, + }, "bigcode/starcoder2-15b": { "name": "StarCoder2-15B", "link": "https://huggingface.co/bigcode/starcoder2-15b", @@ -43,7 +47,8 @@ "size": 15, "act_param": 15, "open-data": "Full", - }, + "reasoning": False, + }, "Qwen/CodeQwen1.5-7B": { "name": "CodeQwen1.5-7B", "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B", @@ -52,7 +57,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "google/codegemma-2b": { "name": "CodeGemma-2B", "link": "https://huggingface.co/google/codegemma-2b", @@ -61,7 +67,8 @@ "size": 2, "act_param": 2, "open-data": "None", - }, + "reasoning": False, + }, "google/codegemma-7b": { "name": "CodeGemma-7B", "link": "https://huggingface.co/google/codegemma-7b", @@ -70,7 +77,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "google/codegemma-7b-it": { "name": "CodeGemma-7B-Instruct", "link": "https://huggingface.co/google/codegemma-7b-it", @@ -79,7 +87,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "gpt-3.5-turbo-0125": { "name": "GPT-3.5-Turbo-0125", "link": "https://openai.com/index/new-embedding-models-and-api-updates", @@ -88,7 +97,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "gpt-4o": { "name": "GPT-4o-2024-05-13", "link": "https://openai.com/index/hello-gpt-4o/", @@ -97,7 +107,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "gpt-4-turbo-2024-04-09": { "name": "GPT-4-Turbo-2024-04-09", "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday", @@ -106,7 +117,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "gpt-4-0613": { "name": "GPT-4-0613", "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday", @@ -115,7 +127,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-7b-hf": { "name": "CodeLlama-7B-Base", "link": "https://huggingface.co/codellama/CodeLlama-7b-hf", @@ -124,7 +137,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-13b-hf": { "name": "CodeLlama-13B-Base", "link": "https://huggingface.co/codellama/CodeLlama-13b-hf", @@ -133,7 +147,8 @@ "size": 13, "act_param": 13, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-7b-Instruct-hf": { "name": "CodeLlama-7B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf", @@ -142,7 +157,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-13b-Instruct-hf": { "name": "CodeLlama-13B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf", @@ -151,7 +167,8 @@ "size": 13, "act_param": 13, "open-data": "None", - }, + "reasoning": False, + }, "mistral-large-2402": { "name": "Mistral-Large-2402", "link": "https://mistral.ai/news/mistral-large/", @@ -160,7 +177,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "mistral-small-2402": { "name": "Mistral-Small-2402", "link": "https://mistral.ai/news/mistral-large/", @@ -169,7 +187,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "mistralai/Mixtral-8x22B-v0.1": { "name": "Mixtral-8x22B-Base", "link": "https://huggingface.co/mistralai/Mixtral-8x22B-v0.1", @@ -178,7 +197,8 @@ "size": 176, "act_param": 44, "open-data": "None", - }, + "reasoning": False, + }, "mistralai/Mixtral-8x22B-Instruct-v0.1": { "name": "Mixtral-8x22B-Instruct", "link": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1", @@ -187,7 +207,8 @@ "size": 176, "act_param": 44, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-34b-hf": { "name": "CodeLlama-34B-Base", "link": "https://huggingface.co/codellama/CodeLlama-34b-hf", @@ -196,7 +217,8 @@ "size": 34, "act_param": 34, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-34b-Instruct-hf": { "name": "CodeLlama-34B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf", @@ -205,7 +227,8 @@ "size": 34, "act_param": 34, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-70b-hf": { "name": "CodeLlama-70B-Base", "link": "https://huggingface.co/codellama/CodeLlama-70b-hf", @@ -214,7 +237,8 @@ "size": 70, "act_param": 70, "open-data": "None", - }, + "reasoning": False, + }, "codellama/CodeLlama-70b-Instruct-hf": { "name": "CodeLlama-70B-Instruct", "link": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf", @@ -223,7 +247,8 @@ "size": 70, "act_param": 70, "open-data": "None", - }, + "reasoning": False, + }, "Qwen/CodeQwen1.5-7B-Chat": { "name": "CodeQwen1.5-7B-Chat", "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat", @@ -232,7 +257,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "Qwen/Qwen1.5-110B-Chat": { "name": "Qwen1.5-110B-Chat", "link": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat", @@ -241,7 +267,8 @@ "size": 110, "act_param": 110, "open-data": "None", - }, + "reasoning": False, + }, "Qwen/Qwen1.5-72B-Chat": { "name": "Qwen1.5-72B-Chat", "link": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat", @@ -250,7 +277,8 @@ "size": 72, "act_param": 72, "open-data": "None", - }, + "reasoning": False, + }, "Qwen/Qwen1.5-32B-Chat": { "name": "Qwen1.5-32B-Chat", "link": "https://huggingface.co/Qwen/Qwen1.5-32B-Chat", @@ -259,7 +287,8 @@ "size": 32, "act_param": 32, "open-data": "None", - }, + "reasoning": False, + }, "deepseek-ai/DeepSeek-V2-Chat": { "name": "DeepSeek-V2-Chat", "link": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat", @@ -268,7 +297,8 @@ "size": 236, "act_param": 21, "open-data": "None", - }, + "reasoning": False, + }, "deepseek-ai/deepseek-coder-1.3b-base": { "name": "DeepSeek-Coder-1.3B-Base", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base", @@ -277,7 +307,8 @@ "size": 1.3, "act_param": 1.3, "open-data": "None", - }, + "reasoning": False, + }, "deepseek-ai/deepseek-coder-1.3b-instruct": { "name": "DeepSeek-Coder-1.3B-Instruct", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct", @@ -286,7 +317,8 @@ "size": 1.3, "act_param": 1.3, "open-data": "None", - }, + "reasoning": False, + }, "deepseek-ai/deepseek-coder-33b-base": { "name": "DeepSeek-Coder-33B-Base", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base", @@ -295,7 +327,8 @@ "size": 33, "act_param": 33, "open-data": "None", - }, + "reasoning": False, + }, "deepseek-ai/deepseek-coder-33b-instruct": { "name": "DeepSeek-Coder-33B-Instruct", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct", @@ -304,7 +337,8 @@ "size": 33, "act_param": 33, "open-data": "None", - }, + "reasoning": False, + }, "deepseek-ai/deepseek-coder-6.7b-base": { "name": "DeepSeek-Coder-6.7B-Base", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", @@ -313,7 +347,8 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - }, + "reasoning": False, + }, "deepseek-ai/deepseek-coder-6.7b-instruct": { "name": "DeepSeek-Coder-6.7B-Instruct", "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct", @@ -322,7 +357,8 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - }, + "reasoning": False, + }, "meta-llama/Meta-Llama-3-70B": { "name": "Llama-3-70B-Base", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B", @@ -331,7 +367,8 @@ "size": 70, "act_param": 70, "open-data": "None", - }, + "reasoning": False, + }, "meta-llama/Meta-Llama-3-70B-Instruct": { "name": "Llama-3-70B-Instruct", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct", @@ -340,7 +377,8 @@ "size": 70, "act_param": 70, "open-data": "None", - }, + "reasoning": False, + }, "meta-llama/Meta-Llama-3-8B": { "name": "Llama-3-8B-Base", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", @@ -349,7 +387,8 @@ "size": 8, "act_param": 8, "open-data": "None", - }, + "reasoning": False, + }, "meta-llama/Meta-Llama-3-8B-Instruct": { "name": "Llama-3-8B-Instruct", "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct", @@ -358,7 +397,8 @@ "size": 8, "act_param": 8, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-3b-code-instruct": { "name": "Granite-Code-3B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-3b-code-instruct", @@ -367,7 +407,8 @@ "size": 3, "act_param": 3, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-8b-code-instruct": { "name": "Granite-Code-8B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-8b-code-instruct", @@ -376,7 +417,8 @@ "size": 8, "act_param": 8, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-20b-code-instruct": { "name": "Granite-Code-20B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-20b-code-instruct", @@ -385,7 +427,8 @@ "size": 20, "act_param": 20, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-34b-code-instruct": { "name": "Granite-Code-34B-Instruct", "link": "https://huggingface.co/ibm-granite/granite-34b-code-instruct", @@ -394,7 +437,8 @@ "size": 34, "act_param": 34, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-3b-code-base": { "name": "Granite-Code-3B-Base", "link": "https://huggingface.co/ibm-granite/granite-3b-code-base", @@ -403,7 +447,8 @@ "size": 3, "act_param": 3, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-8b-code-base": { "name": "Granite-Code-8B-Base", "link": "https://huggingface.co/ibm-granite/granite-8b-code-base", @@ -412,7 +457,8 @@ "size": 8, "act_param": 8, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-20b-code-base": { "name": "Granite-Code-20B-Base", "link": "https://huggingface.co/ibm-granite/granite-20b-code-base", @@ -421,7 +467,8 @@ "size": 20, "act_param": 20, "open-data": "None", - }, + "reasoning": False, + }, "ibm-granite/granite-34b-code-base": { "name": "Granite-Code-34B-Base", "link": "https://huggingface.co/ibm-granite/granite-34b-code-base", @@ -430,7 +477,8 @@ "size": 34, "act_param": 34, "open-data": "None", - }, + "reasoning": False, + }, "claude-3-haiku-20240307": { "name": "Claude-3-Haiku-20240307", "link": "https://www.anthropic.com/news/claude-3-family", @@ -439,7 +487,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": True, + }, "claude-3-sonnet-20240229": { "name": "Claude-3-Sonnet-20240229", "link": "https://www.anthropic.com/news/claude-3-family", @@ -448,7 +497,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": True, + }, "claude-3-opus-20240229": { "name": "Claude-3-Opus-20240229", "link": "https://www.anthropic.com/news/claude-3-family", @@ -457,7 +507,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": True, + }, "01-ai/Yi-1.5-34B-Chat": { "name": "Yi-1.5-34B-Chat", "link": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat", @@ -466,7 +517,8 @@ "size": 34, "act_param": 34, "open-data": "None", - }, + "reasoning": False, + }, "01-ai/Yi-1.5-34B": { "name": "Yi-1.5-34B", "link": "https://huggingface.co/01-ai/Yi-1.5-34B", @@ -475,7 +527,8 @@ "size": 34, "act_param": 34, "open-data": "None", - }, + "reasoning": False, + }, "01-ai/Yi-1.5-9B-Chat": { "name": "Yi-1.5-9B-Chat", "link": "https://huggingface.co/01-ai/Yi-1.5-9B-Chat", @@ -484,7 +537,8 @@ "size": 9, "act_param": 9, "open-data": "None", - }, + "reasoning": False, + }, "01-ai/Yi-1.5-9B": { "name": "Yi-1.5-9B", "link": "https://huggingface.co/01-ai/Yi-1.5-9B", @@ -493,7 +547,8 @@ "size": 9, "act_param": 9, "open-data": "None", - }, + "reasoning": False, + }, "01-ai/Yi-1.5-6B-Chat": { "name": "Yi-1.5-6B-Chat", "link": "https://huggingface.co/01-ai/Yi-1.5-6B-Chat", @@ -502,7 +557,8 @@ "size": 6, "act_param": 6, "open-data": "None", - }, + "reasoning": False, + }, "01-ai/Yi-1.5-6B": { "name": "Yi-1.5-6B", "link": "https://huggingface.co/01-ai/Yi-1.5-6B", @@ -511,7 +567,8 @@ "size": 6, "act_param": 6, "open-data": "None", - }, + "reasoning": False, + }, "Qwen/Qwen2-57B-A14B": { "name": "Qwen2-57B-A14B", "link": "https://huggingface.co/Qwen/Qwen2-57B-A14B", @@ -520,7 +577,8 @@ "size": 57, "act_param": 14, "open-data": "None", - }, + "reasoning": False, + }, "Qwen/Qwen2-7B-Instruct": { "name": "Qwen2-7B-Instruct", "link": "https://huggingface.co/Qwen/Qwen2-7B-Instruct", @@ -529,7 +587,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "Qwen/Qwen2-72B-Chat": { "name": "Qwen2-72B-Chat", "link": "https://huggingface.co/Qwen/Qwen2-72B-Chat", @@ -538,7 +597,8 @@ "size": 72, "act_param": 72, "open-data": "None", - }, + "reasoning": False, + }, "gemini-1.5-pro": { "name": "Gemini-1.5-Pro-API-0514", "link": "https://deepmind.google/technologies/gemini/pro", @@ -547,7 +607,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "gemini-1.5-flash": { "name": "Gemini-1.5-Flash-API-0514", "link": "https://deepmind.google/technologies/gemini/flash/", @@ -556,7 +617,8 @@ "size": None, "act_param": None, "open-data": "None", - }, + "reasoning": False, + }, "m-a-p/OpenCodeInterpreter-DS-33B": { "name": "OpenCodeInterpreter-DS-33B", "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B", @@ -565,7 +627,8 @@ "size": 33, "act_param": 33, "open-data": "Partial", - }, + "reasoning": False, + }, "m-a-p/OpenCodeInterpreter-DS-6.7B": { "name": "OpenCodeInterpreter-DS-6.7B", "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B", @@ -574,7 +637,8 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", - }, + "reasoning": False, + }, "m-a-p/OpenCodeInterpreter-DS-1.3B": { "name": "OpenCodeInterpreter-DS-1.3B", "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B", @@ -583,7 +647,8 @@ "size": 1.3, "act_param": 1.3, "open-data": "Partial", - }, + "reasoning": False, + }, "microsoft/Phi-3-medium-128k-instruct": { "name": "Phi-3-Medium-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct", @@ -592,7 +657,8 @@ "size": 14, "act_param": 14, "open-data": "None", - }, + "reasoning": False, + }, "microsoft/Phi-3-small-128k-instruct": { "name": "Phi-3-Small-128K-Instruct", "link": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct", @@ -601,7 +667,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "codestral-2405": { "name": "Codestral-22B-v0.1", "link": "https://huggingface.co/mistralai/Codestral-22B-v0.1", @@ -610,7 +677,8 @@ "size": 22, "act_param": 22, "open-data": "None", - }, + "reasoning": False, + }, "codestral-mamba-2407": { "name": "Codestral-Mamba", "link": "https://huggingface.co/mistralai/mamba-codestral-7B-v0.1", @@ -619,7 +687,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "mistralai/Mistral-7B-Instruct-v0.3": { "name": "Mistral-7B-Instruct-v0.3", "link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3", @@ -628,7 +697,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "mistralai/Mistral-7B-v0.3": { "name": "Mistral-7B-v0.3", "link": "https://huggingface.co/mistralai/Mistral-7B-v0.3", @@ -637,7 +707,8 @@ "size": 7, "act_param": 7, "open-data": "None", - }, + "reasoning": False, + }, "CohereForAI/c4ai-command-r-plus": { "name": "Command R+", "link": "https://huggingface.co/CohereForAI/c4ai-command-r-plus", @@ -646,6 +717,7 @@ "size": 104, "act_param": 104, "open-data": "None", + "reasoning": False, }, "deepseek-coder": { "name": "DeepSeek-Coder-V2-Instruct", @@ -655,6 +727,7 @@ "size": 236, "act_param": 21, "open-data": "None", + "reasoning": True, }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": { "name": "DeepSeek-Coder-V2-Lite-Instruct", @@ -664,6 +737,7 @@ "size": 16, "act_param": 2.4, "open-data": "None", + "reasoning": False, }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Base": { "name": "DeepSeek-Coder-V2-Lite-Base", @@ -673,6 +747,7 @@ "size": 16, "act_param": 2.4, "open-data": "None", + "reasoning": False, }, "claude-3-5-sonnet-20240620": { "name": "Claude-3.5-Sonnet-20240620", @@ -682,6 +757,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "NousResearch/Hermes-2-Theta-Llama-3-70B": { "name": "Hermes-2-Theta-Llama-3-70B", @@ -691,6 +767,7 @@ "size": 70, "act_param": 70, "open-data": "None", + "reasoning": False, }, "microsoft/wavecoder-ultra-6.7b": { "name": "WaveCoder-Ultra-6.7B", @@ -700,6 +777,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", + "reasoning": False, }, "google/gemma-2-9b-it": { "name": "Gemma-2-9B-Instruct", @@ -709,6 +787,7 @@ "size": 9, "act_param": 9, "open-data": "None", + "reasoning": False, }, "Bin12345/AutoCoder": { "name": "AutoCoder", @@ -718,6 +797,7 @@ "size": 33, "act_param": 33, "open-data": "None", + "reasoning": False, }, "Bin12345/AutoCoder_S_6.7B": { "name": "AutoCoder-S-6.7B", @@ -727,6 +807,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", + "reasoning": False, }, "Bin12345/AutoCoder_QW_7B": { "name": "AutoCoder-QW-7B", @@ -736,6 +817,7 @@ "size": 7, "act_param": 7, "open-data": "None", + "reasoning": False, }, "SenseLLM/ReflectionCoder-DS-33B": { "name": "ReflectionCoder-DS-33B", @@ -745,6 +827,7 @@ "size": 33, "act_param": 33, "open-data": "Partial", + "reasoning": False, }, "SenseLLM/ReflectionCoder-DS-6.7B": { "name": "ReflectionCoder-DS-6.7B", @@ -754,6 +837,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", + "reasoning": False, }, "SenseLLM/ReflectionCoder-CL-34B": { "name": "ReflectionCoder-CL-34B", @@ -763,6 +847,7 @@ "size": 34, "act_param": 34, "open-data": "Partial", + "reasoning": False, }, "SenseLLM/ReflectionCoder-CL-7B": { "name": "ReflectionCoder-CL-7B", @@ -772,6 +857,7 @@ "size": 7, "act_param": 7, "open-data": "Partial", + "reasoning": False, }, "new-microsoft/Phi-3-mini-128k-instruct": { "name": "Phi-3.1-Mini-128K-Instruct", @@ -781,6 +867,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", + "reasoning": False, }, "old-microsoft/Phi-3-mini-128k-instruct": { "name": "Phi-3-Mini-128K-Instruct", @@ -790,6 +877,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", + "reasoning": False, }, "internlm/internlm2_5-7b-chat": { "name": "InternLM2.5-7B-Chat", @@ -799,6 +887,7 @@ "size": 7, "act_param": 7, "open-data": "None", + "reasoning": False, }, "NousResearch/Hermes-2-Pro-Llama-3-70B": { "name": "Hermes-2-Pro-Llama-3-70B", @@ -808,6 +897,7 @@ "size": 70, "act_param": 70, "open-data": "Partial", + "reasoning": False, }, "new-deepseek-chat": { "name": "DeepSeek-V2-Chat (2024-06-28)", @@ -817,6 +907,7 @@ "size": 236, "act_param": 21, "open-data": "None", + "reasoning": True, }, "vllm-google/gemma-2-27b-it": { "name": "Gemma-2-27B-Instruct", @@ -826,6 +917,7 @@ "size": 27, "act_param": 27, "open-data": "None", + "reasoning": False, }, "Artigenz/Artigenz-Coder-DS-6.7B": { "name": "Artigenz-Coder-DS-6.7B", @@ -835,6 +927,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", + "reasoning": False, }, "openchat/openchat-3.6-8b-20240522": { "name": "OpenChat-3.6-8B-20240522", @@ -844,6 +937,7 @@ "size": 8, "act_param": 8, "open-data": "None", + "reasoning": False, }, "Phind/Phind-CodeLlama-34B-v2": { "name": "Phind-CodeLlama-34B-v2", @@ -853,6 +947,7 @@ "size": 34, "act_param": 34, "open-data": "None", + "reasoning": False, }, "yi-large": { "name": "Yi-Large", @@ -862,6 +957,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "THUDM/codegeex4-all-9b": { "name": "CodeGeex4-All-9B", @@ -871,6 +967,7 @@ "size": 9, "act_param": 9, "open-data": "None", + "reasoning": False, }, "gpt-4o-mini-2024-07-18": { "name": "GPT-4o-mini-2024-07-18", @@ -880,6 +977,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "Nexusflow/Athene-70B": { "name": "Athene-70B", @@ -889,6 +987,7 @@ "size": 70, "act_param": 70, "open-data": "None", + "reasoning": False, }, "NTQAI/Nxcode-CQ-7B-orpo": { "name": "Nxcode-CQ-7B-Orpo", @@ -898,6 +997,7 @@ "size": 7, "act_param": 7, "open-data": "None", + "reasoning": False, }, "migtissera/Llama-3-70B-Synthia-v3.5": { "name": "Llama-3-70B-Synthia-v3.5", @@ -907,6 +1007,7 @@ "size": 70, "act_param": 70, "open-data": "None", + "reasoning": False, }, "migtissera/Tess-v2.5.2-Qwen2-72B": { "name": "Tess-v2.5.2-Qwen2-72B", @@ -916,6 +1017,7 @@ "size": 72, "act_param": 72, "open-data": "None", + "reasoning": False, }, "WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5": { "name": "WhiteRabbitNeo-33B-v1.5", @@ -925,6 +1027,7 @@ "size": 33, "act_param": 33, "open-data": "None", + "reasoning": False, }, "mistral-large-2407": { "name": "Mistral-Large-Instruct-2407", @@ -934,6 +1037,7 @@ "size": 123, "act_param": 123, "open-data": "None", + "reasoning": True, }, "meta-llama/Meta-Llama-3.1-8B-Instruct": { "name": "Llama-3.1-8B-Instruct", @@ -943,6 +1047,7 @@ "size": 8, "act_param": 8, "open-data": "None", + "reasoning": False, }, "meta-llama/Meta-Llama-3.1-70B-Instruct": { "name": "Llama-3.1-70B-Instruct", @@ -952,6 +1057,7 @@ "size": 70, "act_param": 70, "open-data": "None", + "reasoning": False, }, "meta--llama-3.1-405b-instruct": { "name": "Llama-3.1-405B-Instruct", @@ -961,6 +1067,7 @@ "size": 405, "act_param": 405, "open-data": "None", + "reasoning": False, }, "deepseek-coder-20240724": { "name": "DeepSeek-Coder-V2-Instruct (2024-07-24)", @@ -970,6 +1077,7 @@ "size": 236, "act_param": 21, "open-data": "None", + "reasoning": False, }, "microsoft/Phi-3.5-mini-instruct": { "name": "Phi-3.5-Mini-Instruct", @@ -979,6 +1087,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", + "reasoning": False, }, "nv-mistralai--mistral-nemo-12b-instruct": { "name": "Mistral-Nemo-12B-Instruct", @@ -988,6 +1097,7 @@ "size": 12, "act_param": 12, "open-data": "None", + "reasoning": False, }, "wyt2000/InverseCoder-CL-13B": { "name": "InverseCoder-CL-13B", @@ -997,6 +1107,7 @@ "size": 13, "act_param": 13, "open-data": "Partial", + "reasoning": False, }, "wyt2000/InverseCoder-CL-7B": { "name": "InverseCoder-CL-7B", @@ -1006,6 +1117,7 @@ "size": 7, "act_param": 7, "open-data": "Partial", + "reasoning": False, }, "wyt2000/InverseCoder-DS-6.7B": { "name": "InverseCoder-DS-6.7B", @@ -1015,6 +1127,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", + "reasoning": False, }, "gemini-1.5-pro-exp-0801": { "name": "Gemini-1.5-Pro-Exp-0801", @@ -1024,6 +1137,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "gpt-4o-2024-08-06": { "name": "GPT-4o-2024-08-06", @@ -1033,6 +1147,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "abacusai/Dracarys-Llama-3.1-70B-Instruct": { "name": "Dracarys-Llama-3.1-70B-Instruct", @@ -1042,6 +1157,7 @@ "size": 70, "act_param": 70, "open-data": "None", + "reasoning": False, }, "abacusai/Dracarys-72B-Instruct": { "name": "Dracarys-72B-Instruct", @@ -1051,6 +1167,7 @@ "size": 72, "act_param": 72, "open-data": "None", + "reasoning": False, }, "gemini-1.5-pro-exp-0827": { "name": "Gemini-1.5-Pro-Exp-0827", @@ -1060,6 +1177,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "gemini-1.5-flash-exp-0827": { "name": "Gemini-1.5-Flash-Exp-0827", @@ -1069,6 +1187,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "microsoft/Phi-3.5-mini-instruct": { "name": "Phi-3.5-Mini-Instruct", @@ -1078,6 +1197,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", + "reasoning": False, }, "abacusai/Dracarys-Llama-3.1-70B-Instruct": { "name": "Dracarys-Llama-3.1-70B-Instruct", @@ -1087,6 +1207,7 @@ "size": 70, "act_param": 70, "open-data": "None", + "reasoning": False, }, "abacusai/Dracarys-72B-Instruct": { "name": "Dracarys-72B-Instruct", @@ -1096,6 +1217,7 @@ "size": 72, "act_param": 72, "open-data": "None", + "reasoning": False, }, "deepseek-coder-v2.5": { "name": "DeepSeek-V2.5", @@ -1105,6 +1227,7 @@ "size": 236, "act_param": 21, "open-data": "None", + "reasoning": False, }, "CohereForAI/c4ai-command-r-08-2024": { "name": "C4AI-Command-R-08-2024", @@ -1114,6 +1237,7 @@ "size": 32.3, "act_param": 32.3, "open-data": "None", + "reasoning": False, }, "CohereForAI/c4ai-command-r-plus-08-2024": { "name": "C4AI-Command-R-Plus-08-2024", @@ -1123,6 +1247,7 @@ "size": 104, "act_param": 104, "open-data": "None", + "reasoning": False, }, "ayueei--yue-coder-9b-preview": { "name": "Yi-Coder-9B-Chat", @@ -1132,6 +1257,7 @@ "size": 9, "act_param": 9, "open-data": "None", + "reasoning": False, }, # "mattshumer/ref_70_e3_prefill": { # "name": "Reflection-Llama-3.1-70B", @@ -1159,6 +1285,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "o1-mini-2024-09-12": { "name": "o1-Mini-2024-09-12 (temperature=1)", @@ -1168,6 +1295,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "Qwen/Qwen2.5-Coder-1.5B-Instruct": { "name": "Qwen2.5-Coder-1.5B-Instruct", @@ -1177,6 +1305,7 @@ "size": 1.5, "act_param": 1.5, "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-Coder-7B-Instruct": { "name": "Qwen2.5-Coder-7B-Instruct", @@ -1186,6 +1315,7 @@ "size": 7, "act_param": 7, "open-data": "None", + "reasoning": False, }, "gemini-1.5-pro-002": { "name": "Gemini-1.5-Pro-002", @@ -1195,6 +1325,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "mistralai/Mistral-Small-Instruct-2409": { "name": "Mistral-Small-Instruct-2409", @@ -1204,6 +1335,7 @@ "size": 22.2, "act_param": 22.2, "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-0.5B-Instruct": { "name": "Qwen2.5-0.5B-Instruct", @@ -1213,6 +1345,7 @@ "size": 0.5, "act_param": 0.5, "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-1.5B-Instruct": { "name": "Qwen2.5-1.5B-Instruct", @@ -1222,6 +1355,7 @@ "size": 1.5, "act_param": 1.5, "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-7B-Instruct": { "name": "Qwen2.5-7B-Instruct", @@ -1231,6 +1365,7 @@ "size": 7, "act_param": 7, "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-14B-Instruct": { "name": "Qwen2.5-14B-Instruct", @@ -1240,6 +1375,7 @@ "size": 14, "act_param": 14, "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-32B-Instruct": { "name": "Qwen2.5-32B-Instruct", @@ -1249,6 +1385,7 @@ "size": 32, "act_param": 32, "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-72B-Instruct": { "name": "Qwen2.5-72B-Instruct", @@ -1258,6 +1395,7 @@ "size": 72, "act_param": 72, "open-data": "None", + "reasoning": False, }, "meta-llama/Llama-3.2-1B-Instruct": { "name": "Llama-3.2-1B-Instruct", @@ -1267,6 +1405,7 @@ "size": 1, "act_param": 1, "open-data": "None", + "reasoning": False, }, "meta-llama/Llama-3.2-3B-Instruct": { "name": "Llama-3.2-3B-Instruct", @@ -1276,6 +1415,7 @@ "size": 3, "act_param": 3, "open-data": "None", + "reasoning": False, }, "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { "name": "Llama-3.1-Nemotron-70B-Instruct", @@ -1285,6 +1425,7 @@ "size": 70, "act_param": 70, "open-data": "Partial", + "reasoning": False, }, "claude-3-5-sonnet-20241022": { "name": "Claude-3.5-Sonnet-20241022", @@ -1294,6 +1435,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "ibm-granite/granite-3.0-8b-instruct": { "name": "Granite-3.0-8B-Instruct", @@ -1303,6 +1445,7 @@ "size": 8, "act_param": 8, "open-data": "None", + "reasoning": False, }, "ibm-granite/granite-3.0-2b-instruct": { "name": "Granite-3.0-2B-Instruct", @@ -1312,6 +1455,7 @@ "size": 2, "act_param": 2, "open-data": "None", + "reasoning": False, }, "grok-beta--main": { "name": "Grok-Beta", @@ -1321,6 +1465,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "claude-3-5-haiku-20241022--main": { "name": "Claude-3.5-Haiku-20241022", @@ -1330,6 +1475,17 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, + }, + "Qwen/Qwen2.5-Coder-14B-Instruct--main": { + "name": "Qwen2.5-Coder-14B-Instruct", + "link": "https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct", + "prompted": True, + "moe": False, + "size": 14, + "act_param": 14, + "open-data": "None", + "reasoning": False, }, "Qwen/Qwen2.5-Coder-32B-Instruct--main": { "name": "Qwen2.5-Coder-32B-Instruct", @@ -1339,6 +1495,7 @@ "size": 32, "act_param": 32, "open-data": "None", + "reasoning": False, }, "infly/OpenCoder-1.5B-Instruct--main": { "name": "OpenCoder-1.5B-Instruct", @@ -1348,6 +1505,7 @@ "size": 1.5, "act_param": 1.5, "open-data": "None", + "reasoning": False, }, "infly/OpenCoder-8B-Instruct--main": { "name": "OpenCoder-8B-Instruct", @@ -1357,6 +1515,7 @@ "size": 8, "act_param": 8, "open-data": "None", + "reasoning": False, }, "microsoft/Phi-3.5-mini-instruct--main": { "name": "Phi-3.5-Mini-Instruct", @@ -1366,6 +1525,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", + "reasoning": False, }, "Nexusflow/Athene-V2-Agent--main": { "name": "Athene-V2-Agent", @@ -1375,6 +1535,7 @@ "size": 72, "act_param": 72, "open-data": "None", + "reasoning": False, }, "Nexusflow/Athene-V2-Chat--main": { "name": "Athene-V2-Chat", @@ -1384,6 +1545,7 @@ "size": 72, "act_param": 72, "open-data": "None", + "reasoning": False, }, "gemini-exp-1114--main": { "name": "Gemini-Exp-1114", @@ -1393,6 +1555,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "gpt-4o-2024-11-20--main": { "name": "GPT-4o-2024-11-20", @@ -1402,6 +1565,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "gemini-exp-1121--main": { "name": "Gemini-Exp-1121", @@ -1411,6 +1575,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "gemini-exp-1206--main": { "name": "Gemini-Exp-1206", @@ -1420,6 +1585,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "meta-llama--Llama-3.3-70B-Instruct--main": { "name": "Llama-3.3-70B-Instruct", @@ -1429,6 +1595,7 @@ "size": 70, "act_param": 70, "open-data": "None", + "reasoning": False, }, "deepseek-ai--DeepSeek-V2.5-1210--main": { "name": "DeepSeek-V2.5-1210", @@ -1438,6 +1605,7 @@ "size": 236, "act_param": 21, "open-data": "None", + "reasoning": False, }, "gemini-2.0-flash-exp--main": { "name": "Gemini-2.0-Flash-Exp", @@ -1447,6 +1615,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "gemini-2.0-flash-thinking-exp-1219--main": { "name": "Gemini-2.0-Flash-Thinking-Exp-1219", @@ -1456,6 +1625,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": False, }, "o1-2024-12-17--main": { "name": "o1-2024-12-17 (temperature=1, reasoning=medium)", @@ -1465,6 +1635,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "o1-2024-12-17--low--main": { "name": "o1-2024-12-17 (temperature=1, reasoning=low)", @@ -1474,6 +1645,7 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "o1-2024-12-17--high--main": { "name": "o1-2024-12-17 (temperature=1, reasoning=high)", @@ -1483,14 +1655,96 @@ "size": None, "act_param": None, "open-data": "None", + "reasoning": True, }, "deepseek-v3-chat--main": { "name": "DeepSeek-V3-Chat", "link": "https://huggingface.co/deepseek-ai/DeepSeek-V3-Chat", "prompted": True, "moe": True, - "size": 685, - "act_param": None, + "size": 671, + "act_param": 37, + "open-data": "None", + "reasoning": True, + }, + "microsoft--phi-4--main": { + "name": "Phi-4", + "link": "https://huggingface.co/microsoft/phi-4", + "prompted": True, + "moe": False, + "size": 14.7, + "act_param": 14.7, + "open-data": "None", + "reasoning": False, + }, + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B--main": { + "name": "DeepSeek-R1-Distill-Llama-70B", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "prompted": True, + "moe": False, + "size": 70, + "act_param": 70, + "open-data": "None", + "reasoning": False, + }, + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B--main": { + "name": "DeepSeek-R1-Distill-Qwen-32B", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "prompted": True, + "moe": False, + "size": 32, + "act_param": 32, + "open-data": "None", + "reasoning": False, + }, + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B--main": { + "name": "DeepSeek-R1-Distill-Qwen-14B", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "prompted": True, + "moe": False, + "size": 14, + "act_param": 14, + "open-data": "None", + "reasoning": False, + }, + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B--main": { + "name": "DeepSeek-R1-Distill-Llama-8B", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "prompted": True, + "moe": False, + "size": 14, + "act_param": 14, "open-data": "None", + "reasoning": False, }, + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B--main": { + "name": "DeepSeek-R1-Distill-Qwen-7B", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + "prompted": True, + "moe": False, + "size": 14, + "act_param": 14, + "open-data": "None", + "reasoning": False, + }, + "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B--main": { + "name": "DeepSeek-R1-Distill-Qwen-1.5B", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + "prompted": True, + "moe": False, + "size": 1.5, + "act_param": 1.5, + "open-data": "None", + "reasoning": False, + }, + "mistralai/Mistral-Small-24B-Instruct-2501--main": { + "name": "Mistral-Small-24B-Instruct-2501", + "link": "https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501", + "prompted": True, + "moe": False, + "size": 24, + "act_param": 24, + "open-data": "None", + "reasoning": False, + } } \ No newline at end of file From 1a340780673fff6c639caaffb79e3c8d8c6d6caf Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 31 Jan 2025 23:15:23 +0800 Subject: [PATCH 261/325] update doc --- ADVANCED_USAGE.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 14c38f3..f271c50 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -59,8 +59,9 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--execution`: The execution backend to use, default to `gradio`. You can choose from `e2b`, `gradio`, `local`. - `--reasoning_effort`: The reasoning effort to use, default to `medium`. You can choose from `easy`, `medium`, `hard` for `o1`, `o3` and `deepseek-reasoner`(soon) models. - `--base_url`: The base URL of the backend for OpenAI-compatible APIs, default to `None` -- `--instruction_prefix`: The instruction prefix for the Anthropic backend, default to `None` -- `--response_prefix`: The response prefix for the Anthropic backend, default to `None` +- `--instruction_prefix`: The instruction prefix for the Anthropic backend. +- `--response_prefix`: The response prefix for the Anthropic backend. +- `--skip_prefill`: Whether to skip the prefill for vLLM and HF backend, which is useful for reasoning models. - `--revision`: The revision of the model with the vLLM or HF backend, default to `main` - `--tp`: The tensor parallel size for the vLLM backend, default to `1` - `--trust_remote_code`: Whether to trust the remote code, default to `False` From dcff46f86eac3b2da13ab519fb60308e3849a1b2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sat, 1 Feb 2025 09:13:42 +0800 Subject: [PATCH 262/325] fix prefill arg --- bigcodebench/provider/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py index d363533..9c5b48f 100644 --- a/bigcodebench/provider/utility.py +++ b/bigcodebench/provider/utility.py @@ -28,8 +28,8 @@ def make_raw_chat_prompt( split: str, instruction_prefix: str, response_prefix: str, - prefill: bool, tokenizer: AutoTokenizer, + prefill: bool = True, direct_completion: bool = False, ) -> str: # directly return prompt if it does not have a tokenizer.chat_template From 0c97a4d8c1d525d9fe01f0f9be3ac31ef2f5d487 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 5 Feb 2025 03:44:15 +0800 Subject: [PATCH 263/325] add models --- analysis/utils.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/analysis/utils.py b/analysis/utils.py index c0f2a64..f357728 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1627,6 +1627,16 @@ "open-data": "None", "reasoning": False, }, + "gemini-2.0-flash-thinking-exp-01-21--main": { + "name": "Gemini-2.0-Flash-Thinking-Exp-01-21", + "link": "https://deepmind.google/technologies/gemini", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": False, + }, "o1-2024-12-17--main": { "name": "o1-2024-12-17 (temperature=1, reasoning=medium)", "link": "https://openai.com/o1/", @@ -1677,6 +1687,16 @@ "open-data": "None", "reasoning": False, }, + "deepseek-reasoner--main": { + "name": "DeepSeek-R1", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1", + "prompted": True, + "moe": True, + "size": 671, + "act_param": 37, + "open-data": "None", + "reasoning": True, + }, "deepseek-ai/DeepSeek-R1-Distill-Llama-70B--main": { "name": "DeepSeek-R1-Distill-Llama-70B", "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B", @@ -1746,5 +1766,35 @@ "act_param": 24, "open-data": "None", "reasoning": False, - } + }, + "o3-mini-2025-01-31--medium--main": { + "name": "o3-mini-2025-01-31 (temperature=1, reasoning=medium)", + "link": "https://openai.com/index/openai-o3-mini/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": True, + }, + "o3-mini-2025-01-31--low--main": { + "name": "o3-mini-2025-01-31 (temperature=1, reasoning=low)", + "link": "https://openai.com/index/openai-o3-mini/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": True, + }, + "o3-mini-2025-01-31--high--main": { + "name": "o3-mini-2025-01-31 (temperature=1, reasoning=high)", + "link": "https://openai.com/index/openai-o3-mini/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": True, + }, } \ No newline at end of file From fb46c011100f93759435bfaa35583ecff9793956 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 5 Feb 2025 04:30:02 +0800 Subject: [PATCH 264/325] add swe arena --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 2c0fc55..fbad3dd 100755 --- a/README.md +++ b/README.md @@ -25,6 +25,17 @@ 📜 Citation

+

+ 🎉 Check out our latest work +
+ SWE Arena +
+ Open Evaluation Platform on AI for Software Engineering +
+ 100% free to use the latest frontier models! +
+

+ ## 💥 Impact BigCodeBench has been trusted by many LLM teams including: - Zhipu AI From 0331489b29cbf2653b4669597ef431e158882aab Mon Sep 17 00:00:00 2001 From: Terry Yue Zhuo Date: Wed, 5 Feb 2025 04:40:23 +0800 Subject: [PATCH 265/325] Update README.md --- README.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fbad3dd..4bcea25 100755 --- a/README.md +++ b/README.md @@ -25,16 +25,12 @@ 📜 Citation

-

- 🎉 Check out our latest work -
- SWE Arena -
- Open Evaluation Platform on AI for Software Engineering -
- 100% free to use the latest frontier models! -
-

+
+

🎉 Check out our latest work!
+ 🌟 SWE Arena 🌟
+ 🚀 Open Evaluation Platform on AI for Software Engineering 🚀
+ ✨ 100% free to use the latest frontier models! ✨

+
## 💥 Impact BigCodeBench has been trusted by many LLM teams including: From 768177eaa7c863731c3d324d241e373e1c7640c9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 7 Feb 2025 17:34:40 +0800 Subject: [PATCH 266/325] add more models --- analysis/utils.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/analysis/utils.py b/analysis/utils.py index f357728..430e113 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1797,4 +1797,74 @@ "open-data": "None", "reasoning": True, }, -} \ No newline at end of file + "gemini-2.0-flash-001--main": { + "name": "Gemini-2.0-Flash-001", + "link": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": False, + }, + "gemini-2.0-flash-exp--main": { + "name": "Gemini-2.0-Flash-Exp", + "link": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": False, + }, + "gemini-2.0-flash-lite-preview-02-05--main": { + "name": "Gemini-2.0-Flash-Lite-Preview-02-05", + "link": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": False, + }, + "gemini-2.0-pro-exp-02-05--main": { + "name": "Gemini-2.0-Pro-Exp-02-05", + "link": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": False, + }, + "NovaSky-AI--Sky-T1-32B-Flash--main": { + "name": "Sky-T1-32B-Flash", + "link": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Flash", + "prompted": True, + "moe": False, + "size": 32, + "act_param": 32, + "open-data": "None", + "reasoning": False, + }, + "NovaSky-AI--Sky-T1-32B-Preview--main": { + "name": "Sky-T1-32B-Preview", + "link": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Preview", + "prompted": True, + "moe": False, + "size": 32, + "act_param": 32, + "open-data": "None", + "reasoning": False, + }, + "Qwen--QwQ-32B-Preview--main": { + "name": "QwQ-32B-Preview", + "link": "https://huggingface.co/Qwen/QwQ-32B-Preview", + "prompted": True, + "moe": False, + "size": 32, + "act_param": 32, + "open-data": "None", + "reasoning": False, + }, +} From 202203e2a8238156161985e033d8b4a188cd8405 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 7 Feb 2025 17:39:04 +0800 Subject: [PATCH 267/325] update execution doc --- ADVANCED_USAGE.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index f271c50..4f48eca 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -69,7 +69,6 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--tokenizer_legacy`: Whether to use the legacy tokenizer, default to `False` - `--samples`: The path to the generated samples file, default to `None` - `--no_execute`: Whether to not execute the samples, default to `False` -- `--local_execute`: Whether to execute the samples locally, default to `False` - `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page - `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` - `--calibrated`: Whether to use the calibrated samples, default to `True` @@ -208,10 +207,10 @@ You are strongly recommended to use a sandbox such as [docker](https://docs.dock # If you want to change the RAM data segment limit (in MB, 30 GB by default): `--max-data-limit` # If you want to change the RAM stack limit (in MB, 10 MB by default): `--max-stack-limit` # If you want to increase the execution time limit (in seconds, 240 seconds by default): `--min-time-limit` -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --execution local --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl # If you only want to check the ground truths -docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --check-gt-only +docker run -v $(pwd):/app bigcodebench/bigcodebench-evaluate:latest --execution local --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --check-gt-only ``` ...Or if you want to try it locally regardless of the risks ⚠️: @@ -226,11 +225,11 @@ Then, run the evaluation: ```bash # ...Or locally ⚠️ -bigcodebench.evaluate --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl +bigcodebench.evaluate --execution local --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl # ...If you really don't want to check the ground truths -bigcodebench.evaluate --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt +bigcodebench.evaluate --execution local --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --no-gt # If you want to save the pass rate to a file -bigcodebench.evaluate --local_execute --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --save_pass_rate +bigcodebench.evaluate --execution local --split [complete|instruct] --subset [full|hard] --samples samples-sanitized-calibrated.jsonl --save_pass_rate # You are strongly recommended to use the following command to clean up the environment after evaluation: pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; From e940724593593c12f20a5a865a65d355a190c2bb Mon Sep 17 00:00:00 2001 From: fly_dust Date: Sat, 8 Feb 2025 00:29:37 -0800 Subject: [PATCH 268/325] fix make_raw_chat_prompt when prefill is disabled --- bigcodebench/provider/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py index 9c5b48f..ad09159 100644 --- a/bigcodebench/provider/utility.py +++ b/bigcodebench/provider/utility.py @@ -72,7 +72,7 @@ def make_raw_chat_prompt( [ {"role": "user", "content": task_prompt}, ], - tokenize=False, + tokenize=False, add_generation_prompt=True ).split(_MAGIC_SPLITTER_)[0] return task_prompt From 23d4390193fd77a37320725100c2ff02314fee2a Mon Sep 17 00:00:00 2001 From: Chengyu Dong Date: Mon, 10 Feb 2025 17:14:31 -0800 Subject: [PATCH 269/325] add unique cache directory before each code execution --- bigcodebench/gen/util/__init__.py | 109 +++++++++++++++++------------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/bigcodebench/gen/util/__init__.py b/bigcodebench/gen/util/__init__.py index d8088ad..59e5158 100644 --- a/bigcodebench/gen/util/__init__.py +++ b/bigcodebench/gen/util/__init__.py @@ -3,6 +3,7 @@ import sys import types import unittest +import tempfile import multiprocessing from multiprocessing import Array, Value, Manager from bigcodebench.eval.utils import ( @@ -17,55 +18,69 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_stack_limit, times): """Execute trusted code in place.""" + # Specify a unique cache dir by modifying XDG_CONFIG_HOME + old_xdg = os.environ.get("XDG_CONFIG_HOME") + temp_xdg = tempfile.mkdtemp(prefix="xdg_config_") + os.environ["XDG_CONFIG_HOME"] = temp_xdg - with create_tempdir(): - import os - import shutil - import builtins - - rmtree = shutil.rmtree - rmdir = os.rmdir - chdir = os.chdir - module_name = "__test__" - new_module = types.ModuleType(module_name) - reliability_guard(max_as_limit, max_data_limit, max_stack_limit) - # Set necessary attributes for the module - new_module.__dict__.update({ - '__builtins__': builtins, - '__file__': f"{module_name}.py", - '__package__': None, - '__doc__': None, - 'sys': sys, - 'os': os, - 'environ': os.environ, - }) - - # Combine the user code and the test code - full_code = code + "\n" + test_code - - # Compile and execute the combined code within the new module - exec(compile(full_code, f"{module_name}.py", 'exec'), - new_module.__dict__) - sys.modules[module_name] = new_module - TestCases = getattr(new_module, 'TestCases') - loader = unittest.TestLoader() - suite = loader.loadTestsFromTestCase(TestCases) - test_result = unittest.TestResult() - start = time.time() - with safe_environment(), swallow_io(), time_limit(seconds=TIMEOUT_LIMIT): - suite.run(test_result) - - errors = test_result.failures + test_result.errors - if len(errors) > 0: - print(errors) - times.value = -1 + try: + with create_tempdir(): + import shutil + import builtins + + rmtree = shutil.rmtree + rmdir = os.rmdir + chdir = os.chdir + module_name = "__test__" + new_module = types.ModuleType(module_name) + + reliability_guard(max_as_limit, max_data_limit, max_stack_limit) + + # Set necessary attributes for the module + new_module.__dict__.update({ + '__builtins__': builtins, + '__file__': f"{module_name}.py", + '__package__': None, + '__doc__': None, + 'sys': sys, + 'os': os, + 'environ': os.environ, + }) + + # Combine the user code and the test code + full_code = code + "\n" + test_code + + # Compile and execute the combined code within the new module + exec(compile(full_code, f"{module_name}.py", 'exec'), + new_module.__dict__) + sys.modules[module_name] = new_module + TestCases = getattr(new_module, 'TestCases') + loader = unittest.TestLoader() + suite = loader.loadTestsFromTestCase(TestCases) + test_result = unittest.TestResult() + start = time.time() + with safe_environment(), swallow_io(), time_limit(seconds=TIMEOUT_LIMIT): + suite.run(test_result) + + errors = test_result.failures + test_result.errors + if len(errors) > 0: + print(errors) + times.value = -1 + else: + times.value = time.time() - start + + # Needed for cleaning up. + shutil.rmtree = rmtree + os.rmdir = rmdir + os.chdir = chdir + + finally: + # Restore the original environment variable + if old_xdg is None: + os.environ.pop("XDG_CONFIG_HOME", None) else: - times.value = time.time() - start - - # Needed for cleaning up. - shutil.rmtree = rmtree - os.rmdir = rmdir - os.chdir = chdir + os.environ["XDG_CONFIG_HOME"] = old_xdg + shutil.rmtree(temp_xdg, ignore_errors=True) def trusted_check_exec(code, inputs): From f254211e7a615f05a860e3c06cf81fc44578946a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 16:57:34 +0800 Subject: [PATCH 270/325] fix for e2b --- bigcodebench/evaluate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index e242ce4..71e7fa4 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -206,7 +206,10 @@ def evaluate( pass_at_k = dict() - passk = [int(k) for k in pass_k.split(",")] + if isinstance(pass_k, str): + passk = [int(k) for k in pass_k.split(",")] + else: + passk = pass_k if parallel < 1: n_workers = max(1, multiprocessing.cpu_count() // 2) From deb41dd0bb1ef7c24237454538866a4d42490543 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 16:58:23 +0800 Subject: [PATCH 271/325] change e2b docker --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index c6ba2ca..bf0c799 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +# ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 From f13783046d08658a8211c6007161c4ab1b506ca2 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 19:16:09 +0800 Subject: [PATCH 272/325] fix e2b docker --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index bf0c799..c6ba2ca 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -# ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 From 5beceb576a9a40a67bc2f304df1f660929f03533 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 20:03:28 +0800 Subject: [PATCH 273/325] fix e2b --- bigcodebench/evaluate.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 71e7fa4..c8672a5 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -189,6 +189,10 @@ def evaluate( # run the evaluation print(f"Command run in sandbox {e2b_endpoint}") + if not isinstance(pass_k, str): + pass_k = ",".join(map(str, pass_k)) + if not isinstance(selective_evaluate, str): + selective_evaluate = ",".join(map(str, selective_evaluate)) sandbox.commands.run("bigcodebench.evaluate --execution 'local' " f"--split {split} --subset {subset} --samples {samples} " f"--pass_k {pass_k} --save_pass_rate {save_pass_rate} --calibrated {calibrated} " @@ -206,10 +210,7 @@ def evaluate( pass_at_k = dict() - if isinstance(pass_k, str): - passk = [int(k) for k in pass_k.split(",")] - else: - passk = pass_k + passk = [int(k) for k in pass_k.split(",")] if parallel < 1: n_workers = max(1, multiprocessing.cpu_count() // 2) @@ -224,7 +225,10 @@ def evaluate( # Add selective evaluation logic if selective_evaluate: - selected_ids = set(selective_evaluate.split(",")) + if isinstance(selective_evaluate, str): + selected_ids = set(selective_evaluate.split(",")) + else: + selected_ids = set(selective_evaluate) problems = {k: v for k, v in problems.items() if k in selected_ids} if not problems: raise ValueError(f"None of the provided task IDs {selected_ids} were found in the dataset") From 0f4cf18821c001087daa7d5e35a6362ed309045f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 20:03:39 +0800 Subject: [PATCH 274/325] fix e2b --- bigcodebench/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index c8672a5..e005c5d 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -190,9 +190,9 @@ def evaluate( # run the evaluation print(f"Command run in sandbox {e2b_endpoint}") if not isinstance(pass_k, str): - pass_k = ",".join(map(str, pass_k)) + pass_k = ",".join(pass_k) if not isinstance(selective_evaluate, str): - selective_evaluate = ",".join(map(str, selective_evaluate)) + selective_evaluate = ",".join(selective_evaluate) sandbox.commands.run("bigcodebench.evaluate --execution 'local' " f"--split {split} --subset {subset} --samples {samples} " f"--pass_k {pass_k} --save_pass_rate {save_pass_rate} --calibrated {calibrated} " From 6f1c33d2853294acb04f4332e192a21bad0ea98c Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 20:14:17 +0800 Subject: [PATCH 275/325] fix e2b --- bigcodebench/evaluate.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index e005c5d..16bd67b 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -189,10 +189,6 @@ def evaluate( # run the evaluation print(f"Command run in sandbox {e2b_endpoint}") - if not isinstance(pass_k, str): - pass_k = ",".join(pass_k) - if not isinstance(selective_evaluate, str): - selective_evaluate = ",".join(selective_evaluate) sandbox.commands.run("bigcodebench.evaluate --execution 'local' " f"--split {split} --subset {subset} --samples {samples} " f"--pass_k {pass_k} --save_pass_rate {save_pass_rate} --calibrated {calibrated} " @@ -210,8 +206,15 @@ def evaluate( pass_at_k = dict() - passk = [int(k) for k in pass_k.split(",")] - + if isinstance(pass_k, str): + passk = [int(k) for k in pass_k.split(",")] + else: + passk = pass_k + if isinstance(selective_evaluate, str): + selected_ids = set(selective_evaluate.split(",")) + else: + selected_ids = set(selective_evaluate) + if parallel < 1: n_workers = max(1, multiprocessing.cpu_count() // 2) else: From 8b79fc446a7cdefc0a159a16a7067c1f2227caa4 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 21:09:30 +0800 Subject: [PATCH 276/325] fix --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index c6ba2ca..bf0c799 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +# ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 From 75469f4631be357f4d47996b8d753b338b5fd630 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 21:31:14 +0800 Subject: [PATCH 277/325] fix --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index bf0c799..c6ba2ca 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -# ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 From 9c5726aab90262d91dcafd23daae34bc1598cb03 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 21:32:44 +0800 Subject: [PATCH 278/325] fix --- bigcodebench/evaluate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 16bd67b..d645eed 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -210,6 +210,7 @@ def evaluate( passk = [int(k) for k in pass_k.split(",")] else: passk = pass_k + if isinstance(selective_evaluate, str): selected_ids = set(selective_evaluate.split(",")) else: From de90e7eda66a256402db7b5a2719a38ea2dc6183 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 21:33:58 +0800 Subject: [PATCH 279/325] fix --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index c6ba2ca..d0e51f4 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -33,7 +33,7 @@ RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 -RUN cd /bigcodebench && \ +RUN cd /bigcodebench && git checkout e2b_debug && \ pip install . --no-deps RUN pip install --timeout 2000 \ From 1dc1e37449eb59d590bf9a69feaa10875d9442f0 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 22:50:33 +0800 Subject: [PATCH 280/325] fix --- bigcodebench/evaluate.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index d645eed..bac8963 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -214,7 +214,10 @@ def evaluate( if isinstance(selective_evaluate, str): selected_ids = set(selective_evaluate.split(",")) else: - selected_ids = set(selective_evaluate) + try: + selected_ids = set(selective_evaluate) + except: + selected_ids = {} if parallel < 1: n_workers = max(1, multiprocessing.cpu_count() // 2) @@ -229,10 +232,6 @@ def evaluate( # Add selective evaluation logic if selective_evaluate: - if isinstance(selective_evaluate, str): - selected_ids = set(selective_evaluate.split(",")) - else: - selected_ids = set(selective_evaluate) problems = {k: v for k, v in problems.items() if k in selected_ids} if not problems: raise ValueError(f"None of the provided task IDs {selected_ids} were found in the dataset") From cb1ddd097274ed857867951492ef0d8f59090f5e Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 22:51:07 +0800 Subject: [PATCH 281/325] fix --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index d0e51f4..d11653d 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +# ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 From 035221bd2536f0ecbec88fe710f141f8c01f7fea Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Thu, 13 Feb 2025 22:52:04 +0800 Subject: [PATCH 282/325] fix --- sandbox-templates/e2b.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index d11653d..512036c 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,8 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -# ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit +RUN echo +ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 From d32f19eed04404a63ab5ee923c6a60956ce542b9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 01:14:09 +0800 Subject: [PATCH 283/325] fix --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index 512036c..daa735a 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -RUN echo +RUN echo 1 ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench From 5d42541ec3b3a72f46e9c3aa85fddf6b67a23368 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 01:56:11 +0800 Subject: [PATCH 284/325] fix --- bigcodebench/evaluate.py | 2 +- sandbox-templates/e2b.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index bac8963..7a59a89 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -217,7 +217,7 @@ def evaluate( try: selected_ids = set(selective_evaluate) except: - selected_ids = {} + selected_ids = "" if parallel < 1: n_workers = max(1, multiprocessing.cpu_count() // 2) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index daa735a..de3489e 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -RUN echo 1 +RUN echo 2 ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench From 5091ff0ef390e99fcdc0b72885142c7ab4852314 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 03:08:40 +0800 Subject: [PATCH 285/325] fix --- bigcodebench/evaluate.py | 2 +- sandbox-templates/e2b.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 7a59a89..d6d061f 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -231,7 +231,7 @@ def evaluate( problems = get_bigcodebench(subset=subset) # Add selective evaluation logic - if selective_evaluate: + if selected_ids: problems = {k: v for k, v in problems.items() if k in selected_ids} if not problems: raise ValueError(f"None of the provided task IDs {selected_ids} were found in the dataset") diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index de3489e..626cbae 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -RUN echo 2 +RUN echo 3 ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench From 0cd61745993f819076733681a40daf445fc970a9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 03:52:17 +0800 Subject: [PATCH 286/325] fix --- bigcodebench/evaluate.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index d6d061f..9e1fd45 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -205,11 +205,7 @@ def evaluate( else: pass_at_k = dict() - - if isinstance(pass_k, str): - passk = [int(k) for k in pass_k.split(",")] - else: - passk = pass_k + passk = list(pass_k) if isinstance(selective_evaluate, str): selected_ids = set(selective_evaluate.split(",")) From 5cfe22b262580a06a6b12354c009b6a0e8ced893 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 03:52:38 +0800 Subject: [PATCH 287/325] fix --- sandbox-templates/e2b.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index 626cbae..d0e51f4 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -28,7 +28,6 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench # Acquire benchmark code to local -RUN echo 3 ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench From 2ee626303974dd60954c384ceca837402815a5ea Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 03:53:40 +0800 Subject: [PATCH 288/325] fix --- sandbox-templates/e2b.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index d0e51f4..c6ba2ca 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -33,7 +33,7 @@ RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench RUN pip install numpy==1.24.3 pyarrow==14.0.1 -RUN cd /bigcodebench && git checkout e2b_debug && \ +RUN cd /bigcodebench && \ pip install . --no-deps RUN pip install --timeout 2000 \ From 81aca07f26a616ed67960be4f280927b79027eb4 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 03:57:26 +0800 Subject: [PATCH 289/325] fix passk --- bigcodebench/evaluate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 9e1fd45..d6d061f 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -205,7 +205,11 @@ def evaluate( else: pass_at_k = dict() - passk = list(pass_k) + + if isinstance(pass_k, str): + passk = [int(k) for k in pass_k.split(",")] + else: + passk = pass_k if isinstance(selective_evaluate, str): selected_ids = set(selective_evaluate.split(",")) From bdc265c1036f498874b6295fded7b6fe3e39b2c6 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 18:15:55 +0800 Subject: [PATCH 290/325] fix: check if selective_evaluate exists --- bigcodebench/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index d6d061f..309a6e1 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -211,7 +211,7 @@ def evaluate( else: passk = pass_k - if isinstance(selective_evaluate, str): + if selective_evaluate and isinstance(selective_evaluate, str): selected_ids = set(selective_evaluate.split(",")) else: try: From 49d2c522a9ed8ef1b144eb697dc91e7ed6bb0eb9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 18:24:07 +0800 Subject: [PATCH 291/325] fix: optional bool args --- bigcodebench/evaluate.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 309a6e1..20f390f 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -189,12 +189,19 @@ def evaluate( # run the evaluation print(f"Command run in sandbox {e2b_endpoint}") - sandbox.commands.run("bigcodebench.evaluate --execution 'local' " - f"--split {split} --subset {subset} --samples {samples} " - f"--pass_k {pass_k} --save_pass_rate {save_pass_rate} --calibrated {calibrated} " - f"--parallel {parallel} --selective_evaluate {selective_evaluate} --min_time_limit {min_time_limit} " + command = "bigcodebench.evaluate --execution 'local' "\ + f"--split {split} --subset {subset} --samples {samples} "\ + f"--pass_k {pass_k} --save_pass_rate {save_pass_rate} --calibrated {calibrated} "\ + f"--parallel {parallel} --selective_evaluate {selective_evaluate} --min_time_limit {min_time_limit} "\ f"--max_as_limit {max_as_limit} --max_data_limit {max_data_limit} --max_stack_limit {max_stack_limit} " - f"--check_gt_only {check_gt_only} --no_gt {no_gt}", on_stderr=lambda x: print(x), on_stdout=lambda x: print(x), timeout=60*50) + + if check_gt_only: + command += f"--check_gt_only " + if no_gt: + command += f"--no_gt " + if no_execute: + command += f"--no_execute " + sandbox.commands.run(command, on_stdout=lambda x: print(x), on_stderr=lambda x: print(x), timeout=60*60) if not check_gt_only: # download the results From 41770a77a620842ae425fdb0fcc0f7e795ff1351 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 14 Feb 2025 20:37:26 +0800 Subject: [PATCH 292/325] force e2b docker update --- sandbox-templates/e2b.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/sandbox-templates/e2b.Dockerfile b/sandbox-templates/e2b.Dockerfile index c6ba2ca..a667880 100644 --- a/sandbox-templates/e2b.Dockerfile +++ b/sandbox-templates/e2b.Dockerfile @@ -27,6 +27,7 @@ RUN adduser --disabled-password --gecos "" bigcodebenchuser RUN rm -rf /bigcodebench +RUN echo 1 # Acquire benchmark code to local ADD "https://api.github.com/repos/bigcode-project/bigcodebench/commits?per_page=1" latest_commit RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench From 2f9f4c84beff5752ecde26e57d908fec981bc209 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 19 Feb 2025 17:12:31 +0800 Subject: [PATCH 293/325] fix: check if prefill --- bigcodebench/provider/hf.py | 3 ++- bigcodebench/provider/vllm.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bigcodebench/provider/hf.py b/bigcodebench/provider/hf.py index a85957d..fbe50e5 100644 --- a/bigcodebench/provider/hf.py +++ b/bigcodebench/provider/hf.py @@ -41,7 +41,8 @@ def __init__( if self.is_direct_completion(): # no chat template self.eos += extra_eos_for_direct_completion(dataset) else: # with chat template - self.eos += ["\n```\n"] + if self.prefill and "```" in self.response_prefix: + self.eos += ["\n```\n"] print(f"{self.eos = }") self.model = AutoModelForCausalLM.from_pretrained(name, **kwargs) diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 5ce67ab..2bd47d6 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -27,7 +27,8 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: if self.is_direct_completion(): self.eos += extra_eos_for_direct_completion(dataset) else: - self.eos += ["\n```\n"] + if self.prefill: + self.eos += ["\n```\n"] self.llm = LLM(model=name, max_model_len=self.max_new_tokens, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) From 8e5cc7e425590b354263dcf78d3abe08e7aad781 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 19 Feb 2025 17:13:58 +0800 Subject: [PATCH 294/325] fix: check ``` in hf prefill --- bigcodebench/provider/vllm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 2bd47d6..cc928e4 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -27,7 +27,7 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: if self.is_direct_completion(): self.eos += extra_eos_for_direct_completion(dataset) else: - if self.prefill: + if self.prefill and "```" in self.response_prefix: self.eos += ["\n```\n"] self.llm = LLM(model=name, max_model_len=self.max_new_tokens, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) From 04b317f19bf05b119c9d3a28655ae26240427517 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Wed, 19 Feb 2025 23:15:49 +0100 Subject: [PATCH 295/325] Fixes for tasks 211 and 215 --- tools/fix_v022.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tools/fix_v022.py b/tools/fix_v022.py index 88e1f05..4cd531f 100644 --- a/tools/fix_v022.py +++ b/tools/fix_v022.py @@ -11,6 +11,47 @@ BIGCODEBENCH_NEW_VERSION = "v0.1.3" def map_ds(sample): + if sample["task_id"] in ["BigCodeBench/211"]: + sample['test'] = sample['test'].replace( +""" + mock_response = MagicMock() + mock_response.content = MOCK_CONTENT +""", +""" + mock_response = MagicMock() + mock_response.content = MOCK_CONTENT + mock_response.status_code = 200 +""" + ) + if sample["task_id"] in ["BigCodeBench/215"]: + sample['test'] = sample['test'].replace( +""" + mock_response = Mock() +""", +""" + mock_response = Mock() + mock_response.status_code = 200 +""" + ) + sample['test'] = sample['test'].replace( +""" + mock_response.text =""", +""" + MOCK_TEXT =""" + ) + sample['test'] = sample['test'].replace( +""" + mock_get.return_value = mock_response +""", +""" + mock_response.text = MOCK_TEXT + mock_response.json = lambda: json.loads(MOCK_TEXT) + mock_get.return_value = mock_response +""" + ) + sample['complete_prompt'] = sample['complete_prompt'].replace("Thif function will raise", "This function will raise") + sample['instruct_prompt'] = sample['instruct_prompt'].replace("Thif function will raise", "This function will raise") + sample['doc_struct'] = sample['doc_struct'].replace("Thif function will raise", "This function will raise") if sample["task_id"] in ["BigCodeBench/1005"]: for k in sample.keys(): sample[k] = sample[k].replace( @@ -28,7 +69,7 @@ def map_ds(sample): hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) ds = ds_dict[BIGCODEBENCH_VERSION] hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] - function_id = [1005] + function_id = [211, 215, 1005] new_ds = ds.map(map_ds) new_ds.to_json("BigCodeBench.jsonl") From afa881cefddcb9df0861ca322028ea0349f84931 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Wed, 19 Feb 2025 23:27:59 +0100 Subject: [PATCH 296/325] Add support for Hugging Face Serverless Inference --- bigcodebench/gen/util/hf_inference_request.py | 34 ++++++++++++ bigcodebench/provider/__init__.py | 13 +++++ bigcodebench/provider/hf_inference.py | 54 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 bigcodebench/gen/util/hf_inference_request.py create mode 100644 bigcodebench/provider/hf_inference.py diff --git a/bigcodebench/gen/util/hf_inference_request.py b/bigcodebench/gen/util/hf_inference_request.py new file mode 100644 index 0000000..fe4aaf3 --- /dev/null +++ b/bigcodebench/gen/util/hf_inference_request.py @@ -0,0 +1,34 @@ +import time + +from huggingface_hub import InferenceClient +from huggingface_hub.inference._generated.types import TextGenerationOutput + + +def make_request( + client: InferenceClient, + message: str, + model: str, + temperature: float, + n: int, + max_new_tokens: int = 2048, +) -> TextGenerationOutput: + response = client.text_generation( + model=model, + prompt=message, + do_sample=False, + max_new_tokens=max_new_tokens, + ) + + return response + + +def make_auto_request(*args, **kwargs) -> TextGenerationOutput: + ret = None + while ret is None: + try: + ret = make_request(*args, **kwargs) + except Exception as e: + print("Unknown error. Waiting...") + print(e) + time.sleep(1) + return ret diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index dbadfd4..c78d870 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -68,6 +68,19 @@ def make_model( tokenizer_name=tokenizer_name, tokenizer_legacy=tokenizer_legacy, ) + elif backend == "hf-inference": + from bigcodebench.provider.hf_inference import HuggingFaceInferenceDecoder + + return HuggingFaceInferenceDecoder( + name=model, + subset=subset, + split=split, + temperature=temperature, + max_new_tokens=max_new_tokens, + direct_completion=direct_completion, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) elif backend == "openai": from bigcodebench.provider.openai import OpenAIChatDecoder diff --git a/bigcodebench/provider/hf_inference.py b/bigcodebench/provider/hf_inference.py new file mode 100644 index 0000000..1737448 --- /dev/null +++ b/bigcodebench/provider/hf_inference.py @@ -0,0 +1,54 @@ +import os +from typing import List +from tqdm import tqdm + +from huggingface_hub import InferenceClient + +from bigcodebench.provider.base import DecoderBase +from bigcodebench.gen.util.hf_inference_request import make_auto_request +from bigcodebench.provider.utility import make_raw_chat_prompt + + +class HuggingFaceInferenceDecoder(DecoderBase): + def __init__(self, name: str, **kwargs): + super().__init__(name, **kwargs) + self.client = InferenceClient( + provider="hf-inference", api_key=os.getenv("HF_INFERENCE_API_KEY") + ) + + def codegen( + self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + if do_sample: + assert self.temperature > 0, "Temperature must be positive for sampling" + + all_outputs = [] + + for prompt in tqdm(prompts): + outputs = [] + message = ( + prompt + if self.is_direct_completion() + else make_raw_chat_prompt( + task_prompt=prompt, + subset=self.subset, + split=self.split, + instruction_prefix=self.instruction_prefix, + response_prefix=self.response_prefix, + tokenizer=None, + ) + ) + ret = make_auto_request( + self.client, + message=message, + model=self.name, + n=num_samples, + temperature=self.temperature, + max_new_tokens=self.max_new_tokens, + ) + outputs.append(ret) + all_outputs.append(outputs) + return all_outputs + + def is_direct_completion(self) -> bool: + return self.direct_completion From e646fe527e8a6100a1344a4bfcc47149a18014dd Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Wed, 19 Feb 2025 23:36:08 +0100 Subject: [PATCH 297/325] Add docs --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4bcea25..94ad2ef 100755 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ bigcodebench.evaluate \ --execution [e2b|gradio|local] \ --split [complete|instruct] \ --subset [full|hard] \ - --backend [vllm|openai|anthropic|google|mistral|hf] + --backend [vllm|openai|anthropic|google|mistral|hf|hf-inference] ``` - All the resulted files will be stored in a folder named `bcb_results`. @@ -177,6 +177,13 @@ Access Gemini APIs from [Google AI Studio](https://aistudio.google.com/) export GOOGLE_API_KEY= ``` +Access the [Hugging Face Serverless Inference API](https://huggingface.co/docs/api-inference/en/index) +```bash +export HF_INFERENCE_API_KEY= +``` + +Please make sure your HF access token has the `Make calls to inference providers` permission. + ## 💻 LLM-generated Code We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard) on the full set: From 45e901f17ac170e5b43fc388960b7c77b76e020c Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Thu, 20 Feb 2025 03:50:02 +0100 Subject: [PATCH 298/325] Reintroduce progress checker from #48 --- bigcodebench/evaluate.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 20f390f..dcb08d4 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -322,14 +322,13 @@ def evaluate( assert len(completion_id) == len(problems), f"Missing problems in samples. Expected {len(problems)} problems, got {len(completion_id)}" def stucking_checker(): - while remainings: - last_size = len(remainings) - time.sleep(240) - if last_size != len(remainings) or len(remainings) == 0: - continue - # Potential stucking - warn("No samples had finished testing in the last 240s") - warn(f"{len(remainings)} samples to be tested: {remainings}") + not_done = futures + while len(not_done) > 0: + done, not_done = wait(not_done, timeout=240, return_when=FIRST_COMPLETED) + + if len(done) == 0: + warn("No samples have finished testing in the last 240s") + warn(f"{len(remainings)} samples to be tested: {remainings}") threading.Thread(target=stucking_checker).start() From 9515dfccf0aa1426cf5448e8729c8576bf997b93 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Thu, 20 Feb 2025 18:02:24 +0100 Subject: [PATCH 299/325] Move changes to new fix file and bump version numbers --- tools/fix_v022.py | 43 +--------------------- tools/fix_v023.py | 91 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 42 deletions(-) create mode 100644 tools/fix_v023.py diff --git a/tools/fix_v022.py b/tools/fix_v022.py index 4cd531f..88e1f05 100644 --- a/tools/fix_v022.py +++ b/tools/fix_v022.py @@ -11,47 +11,6 @@ BIGCODEBENCH_NEW_VERSION = "v0.1.3" def map_ds(sample): - if sample["task_id"] in ["BigCodeBench/211"]: - sample['test'] = sample['test'].replace( -""" - mock_response = MagicMock() - mock_response.content = MOCK_CONTENT -""", -""" - mock_response = MagicMock() - mock_response.content = MOCK_CONTENT - mock_response.status_code = 200 -""" - ) - if sample["task_id"] in ["BigCodeBench/215"]: - sample['test'] = sample['test'].replace( -""" - mock_response = Mock() -""", -""" - mock_response = Mock() - mock_response.status_code = 200 -""" - ) - sample['test'] = sample['test'].replace( -""" - mock_response.text =""", -""" - MOCK_TEXT =""" - ) - sample['test'] = sample['test'].replace( -""" - mock_get.return_value = mock_response -""", -""" - mock_response.text = MOCK_TEXT - mock_response.json = lambda: json.loads(MOCK_TEXT) - mock_get.return_value = mock_response -""" - ) - sample['complete_prompt'] = sample['complete_prompt'].replace("Thif function will raise", "This function will raise") - sample['instruct_prompt'] = sample['instruct_prompt'].replace("Thif function will raise", "This function will raise") - sample['doc_struct'] = sample['doc_struct'].replace("Thif function will raise", "This function will raise") if sample["task_id"] in ["BigCodeBench/1005"]: for k in sample.keys(): sample[k] = sample[k].replace( @@ -69,7 +28,7 @@ def map_ds(sample): hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) ds = ds_dict[BIGCODEBENCH_VERSION] hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] - function_id = [211, 215, 1005] + function_id = [1005] new_ds = ds.map(map_ds) new_ds.to_json("BigCodeBench.jsonl") diff --git a/tools/fix_v023.py b/tools/fix_v023.py new file mode 100644 index 0000000..22b1559 --- /dev/null +++ b/tools/fix_v023.py @@ -0,0 +1,91 @@ +from datasets import load_dataset, Dataset, DatasetDict +from huggingface_hub import HfApi + +import json +import copy + +BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_HARD_HF = "bigcode/bigcodebench-hard" +BIGCODEBENCH_VERSION = "v0.1.3" +BIGCODEBENCH_UPDATE = "bigcode/bcb_update" +BIGCODEBENCH_NEW_VERSION = "v0.1.4" + +def map_ds(sample): + if sample["task_id"] in ["BigCodeBench/211"]: + sample['test'] = sample['test'].replace( +""" + mock_response = MagicMock() + mock_response.content = MOCK_CONTENT +""", +""" + mock_response = MagicMock() + mock_response.content = MOCK_CONTENT + mock_response.status_code = 200 +""" + ) + if sample["task_id"] in ["BigCodeBench/215"]: + sample['test'] = sample['test'].replace( +""" + mock_response = Mock() +""", +""" + mock_response = Mock() + mock_response.status_code = 200 +""" + ) + sample['test'] = sample['test'].replace( +""" + mock_response.text =""", +""" + MOCK_TEXT =""" + ) + sample['test'] = sample['test'].replace( +""" + mock_get.return_value = mock_response +""", +""" + mock_response.text = MOCK_TEXT + mock_response.json = lambda: json.loads(MOCK_TEXT) + mock_get.return_value = mock_response +""" + ) + sample['complete_prompt'] = sample['complete_prompt'].replace("Thif function will raise", "This function will raise") + sample['instruct_prompt'] = sample['instruct_prompt'].replace("Thif function will raise", "This function will raise") + sample['doc_struct'] = sample['doc_struct'].replace("Thif function will raise", "This function will raise") + return sample + +if __name__ == "__main__": + api = HfApi() + ds_dict = load_dataset(BIGCODEBENCH_HF) + hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) + ds = ds_dict[BIGCODEBENCH_VERSION] + hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] + function_id = [211, 215] + + new_ds = ds.map(map_ds) + new_ds.to_json("BigCodeBench.jsonl") + ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds + ds_dict.push_to_hub(BIGCODEBENCH_HF) + + new_hard_ds = hard_ds.map(map_ds) + new_hard_ds.to_json("BigCodeBench-Hard.jsonl") + hard_ds_dict[BIGCODEBENCH_NEW_VERSION] = new_hard_ds + hard_ds_dict.push_to_hub(BIGCODEBENCH_HARD_HF) + + for i in function_id: + old_sample = ds.select([i]) + new_sample = new_ds.select([i]) + old_sample.to_json("old.jsonl") + new_sample.to_json("new.jsonl") + api.upload_file( + path_or_fileobj="old.jsonl", + path_in_repo=f"{i}/old.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) + api.upload_file( + path_or_fileobj="new.jsonl", + path_in_repo=f"{i}/new.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) From d8161d93d3a39395a5db50b71bdde25ac846f32f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 24 Feb 2025 00:47:40 +0800 Subject: [PATCH 300/325] feat: use v0.1.4 dataset --- bigcodebench/data/bigcodebench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 26090f1..87cfdf6 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -14,7 +14,7 @@ BIGCODEBENCH_OVERRIDE_PATH = os.environ.get("BIGCODEBENCH_OVERRIDE_PATH", None) BIGCODEBENCH_HF = "bigcode/bigcodebench" -BIGCODEBENCH_VERSION = "v0.1.3" +BIGCODEBENCH_VERSION = "v0.1.4" def _ready_bigcodebench_path(subset="full", version="default") -> str: if BIGCODEBENCH_OVERRIDE_PATH: From 9059fb84d1188c02edeac4995361656a2fdecbef Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 24 Feb 2025 00:48:39 +0800 Subject: [PATCH 301/325] remove check_gt_only flag --- run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 6242abd..8bfcdd7 100755 --- a/run.sh +++ b/run.sh @@ -10,5 +10,4 @@ bigcodebench.evaluate \ --model $MODEL \ --split $SPLIT \ --subset $SUBSET \ - --backend $BACKEND \ - --check_gt_only \ No newline at end of file + --backend $BACKEND \ No newline at end of file From 89309066c6e4e590c8a20c1392d504cb9e68917a Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 25 Feb 2025 21:31:42 +0800 Subject: [PATCH 302/325] feat: support anthropic extended thinking --- bigcodebench/gen/util/anthropic_request.py | 13 ++++++++++++- bigcodebench/generate.py | 11 ++++++++++- bigcodebench/provider/__init__.py | 7 ++++++- bigcodebench/provider/anthropic.py | 6 +++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/bigcodebench/gen/util/anthropic_request.py b/bigcodebench/gen/util/anthropic_request.py index e53feab..e240dee 100644 --- a/bigcodebench/gen/util/anthropic_request.py +++ b/bigcodebench/gen/util/anthropic_request.py @@ -16,7 +16,18 @@ def make_auto_request(client: anthropic.Client, *args, **kwargs) -> Message: try: signal.signal(signal.SIGALRM, handler) signal.alarm(100) - ret = client.messages.create(*args, **kwargs) + if "reasoning_budget" in kwargs and "reasoning_beta" in kwargs: + ret = client.beta.messages.create( + *args, + **kwargs, + thinking = { + "type": "enabled", + "budget": kwargs["reasoning_budget"], + }, + betas=[kwargs["reasoning_beta"]] + ) + else: + ret = client.messages.create(*args, **kwargs) signal.alarm(0) except anthropic.RateLimitError: print("Rate limit exceeded. Waiting...") diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index bcf1463..9823d0c 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -132,7 +132,11 @@ def run_codegen( temperature: float = 0.0, max_new_tokens: int = 1280, greedy: bool = False, + # openai reasoning_effort: str = "medium", + # anthropic + reasoning_budget: int = 0, + reasoning_beta: str = "output-128k-2025-02-19", strip_newlines: bool = False, direct_completion: bool = False, resume: bool = True, @@ -173,6 +177,8 @@ def run_codegen( temperature=temperature, max_new_tokens=max_new_tokens, reasoning_effort=reasoning_effort, + reasoning_budget=reasoning_budget, + reasoning_beta=reasoning_beta, instruction_prefix=instruction_prefix, response_prefix=response_prefix, prefill=not skip_prefill, @@ -186,8 +192,11 @@ def run_codegen( ) extra = "-" + subset if subset != "full" else "" - if reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): + if backend == "openai" and reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): model = model + f"--{reasoning_effort}" + + if backend == "anthropic" and reasoning_budget and reasoning_beta: + model = model + f"--{reasoning_budget}-{reasoning_beta}" if skip_prefill: identifier = model.replace("/", "--") + "--skip_prefill" + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index c78d870..f76ec29 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -9,8 +9,11 @@ def make_model( dataset: str = "bigcodebench", temperature: float = 0.0, max_new_tokens: int = 1280, - # o1 and o3 only + # openai only reasoning_effort: str = "medium", + # anthropic only + reasoning_budget: int = 0, + reasoning_beta: str = "output-128k-2025-02-19", # instruction model only instruction_prefix: str = None, response_prefix: str = None, @@ -118,6 +121,8 @@ def make_model( split=split, temperature=temperature, max_new_tokens=max_new_tokens, + reasoning_budget=reasoning_budget, + reasoning_beta=reasoning_beta, instruction_prefix=instruction_prefix, response_prefix=response_prefix, ) diff --git a/bigcodebench/provider/anthropic.py b/bigcodebench/provider/anthropic.py index 1969e0c..1612456 100644 --- a/bigcodebench/provider/anthropic.py +++ b/bigcodebench/provider/anthropic.py @@ -9,9 +9,11 @@ from bigcodebench.provider.utility import make_raw_chat_prompt class AnthropicDecoder(DecoderBase): - def __init__(self, name: str, **kwargs) -> None: + def __init__(self, name: str, reasoning_budget: int = 0, reasoning_beta: str = "output-128k-2025-02-19", **kwargs) -> None: super().__init__(name, **kwargs) self.client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_KEY")) + self.reasoning_budget = reasoning_budget + self.reasoning_beta = reasoning_beta def codegen( self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 @@ -43,6 +45,8 @@ def codegen( max_tokens=self.max_new_tokens, temperature=self.temperature, stop_sequences=self.eos, + reasoning_budget=self.reasoning_budget, + reasoning_beta=self.reasoning_beta, ) outputs.append(ret.content[0].text) all_outputs.append(outputs) From c05694cde596c9728664dbab2c8bed5e5ea9c036 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 25 Feb 2025 21:41:23 +0800 Subject: [PATCH 303/325] fix: remove unused args --- bigcodebench/gen/util/anthropic_request.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bigcodebench/gen/util/anthropic_request.py b/bigcodebench/gen/util/anthropic_request.py index e240dee..20ce444 100644 --- a/bigcodebench/gen/util/anthropic_request.py +++ b/bigcodebench/gen/util/anthropic_request.py @@ -17,15 +17,14 @@ def make_auto_request(client: anthropic.Client, *args, **kwargs) -> Message: signal.signal(signal.SIGALRM, handler) signal.alarm(100) if "reasoning_budget" in kwargs and "reasoning_beta" in kwargs: - ret = client.beta.messages.create( - *args, - **kwargs, - thinking = { - "type": "enabled", - "budget": kwargs["reasoning_budget"], - }, - betas=[kwargs["reasoning_beta"]] - ) + kwargs["thinking"] = { + "type": "enabled", + "budget": kwargs["reasoning_budget"], + } + kwargs["betas"] = [kwargs["reasoning_beta"]] + kwargs.pop("reasoning_budget") + kwargs.pop("reasoning_beta") + ret = client.beta.messages.create(*args, **kwargs) else: ret = client.messages.create(*args, **kwargs) signal.alarm(0) From 57eb973f34666067287cbb05e1845e16b87b5e26 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 26 Feb 2025 00:57:31 +0800 Subject: [PATCH 304/325] fix: correctly process anthropic treaming --- bigcodebench/gen/util/anthropic_request.py | 6 ++++-- bigcodebench/provider/anthropic.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bigcodebench/gen/util/anthropic_request.py b/bigcodebench/gen/util/anthropic_request.py index 20ce444..f6d18fd 100644 --- a/bigcodebench/gen/util/anthropic_request.py +++ b/bigcodebench/gen/util/anthropic_request.py @@ -19,12 +19,14 @@ def make_auto_request(client: anthropic.Client, *args, **kwargs) -> Message: if "reasoning_budget" in kwargs and "reasoning_beta" in kwargs: kwargs["thinking"] = { "type": "enabled", - "budget": kwargs["reasoning_budget"], + "budget_tokens": kwargs["reasoning_budget"], } kwargs["betas"] = [kwargs["reasoning_beta"]] kwargs.pop("reasoning_budget") kwargs.pop("reasoning_beta") - ret = client.beta.messages.create(*args, **kwargs) + kwargs.pop("temperature") + if "thinking" in kwargs: + ret = client.beta.messages.create(*args, **kwargs, stream=True) else: ret = client.messages.create(*args, **kwargs) signal.alarm(0) diff --git a/bigcodebench/provider/anthropic.py b/bigcodebench/provider/anthropic.py index 1612456..59aec09 100644 --- a/bigcodebench/provider/anthropic.py +++ b/bigcodebench/provider/anthropic.py @@ -48,7 +48,17 @@ def codegen( reasoning_budget=self.reasoning_budget, reasoning_beta=self.reasoning_beta, ) - outputs.append(ret.content[0].text) + if isinstance(ret, anthropic.Stream): + output = "" + for chunk in ret: + if chunk.type == "content_block_delta": + if chunk.delta.type == "thinking_delta": + output += chunk.delta.thinking + elif chunk.delta.type == "text_delta": + output += chunk.delta.text + outputs.append(output) + else: + outputs.append(ret.content[0].text) all_outputs.append(outputs) return all_outputs From 78dceb21430359efa05c235324e10523453d7d2f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Wed, 26 Feb 2025 01:02:05 +0800 Subject: [PATCH 305/325] fix: only append text output --- bigcodebench/provider/anthropic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigcodebench/provider/anthropic.py b/bigcodebench/provider/anthropic.py index 59aec09..b4a7e43 100644 --- a/bigcodebench/provider/anthropic.py +++ b/bigcodebench/provider/anthropic.py @@ -52,9 +52,9 @@ def codegen( output = "" for chunk in ret: if chunk.type == "content_block_delta": - if chunk.delta.type == "thinking_delta": - output += chunk.delta.thinking - elif chunk.delta.type == "text_delta": + # if chunk.delta.type == "thinking_delta": + # output += chunk.delta.thinking + if chunk.delta.type == "text_delta": output += chunk.delta.text outputs.append(output) else: From 05b7f1f93355f2e64cc3576c4dd1f6c2dbdeab67 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 2 Mar 2025 19:27:56 +0800 Subject: [PATCH 306/325] doc: fix endpoints --- ADVANCED_USAGE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 4f48eca..9bb81b8 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -69,7 +69,8 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--tokenizer_legacy`: Whether to use the legacy tokenizer, default to `False` - `--samples`: The path to the generated samples file, default to `None` - `--no_execute`: Whether to not execute the samples, default to `False` -- `--remote_execute_api`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page +- `--e2b_endpoint`: The API endpoint for remote execution, default to `bigcodebench_evaluator`, you can also use your own E2B API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page +- `--gradio_endpoint`: The API endpoint for remote execution, default to `https://bigcode-bigcodebench-evaluator.hf.space/`, you can also use your own Gradio API endpoint by cloning the [bigcodebench-evaluator](https://huggingface.co/spaces/bigcode/bigcodebench-evaluator) repo and check `Use via API` at the bottom of the HF space page - `--pass_k`: The `k` in `Pass@k`, default to `[1, 5, 10]`, e.g. `--pass_k 1,5,10` will evaluate `Pass@1`, `Pass@5` and `Pass@10` - `--calibrated`: Whether to use the calibrated samples, default to `True` - `--save_pass_rate`: Whether to save the pass rate to a file, default to `True` From 0ecd667f74cd5f789b36e22dc8564f0fc1c09884 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 2 Mar 2025 20:30:39 +0800 Subject: [PATCH 307/325] update the results analysis script --- analysis/get_results.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index fc5aa17..607615a 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -118,12 +118,12 @@ def check_valid(results): def split_gen(): - shutil.rmtree("sanitized_samples", ignore_errors=True) shutil.rmtree("sanitized_calibrated_samples", ignore_errors=True) - os.makedirs("sanitized_samples/complete", exist_ok=True) - os.makedirs("sanitized_samples/instruct", exist_ok=True) - os.makedirs("sanitized_calibrated_samples/complete", exist_ok=True) - os.makedirs("sanitized_calibrated_samples/instruct", exist_ok=True) + os.makedirs("sanitized_calibrated_samples/hard/complete", exist_ok=True) + os.makedirs("sanitized_calibrated_samples/hard/instruct", exist_ok=True) + os.makedirs("sanitized_calibrated_samples/full/complete", exist_ok=True) + os.makedirs("sanitized_calibrated_samples/full/instruct", exist_ok=True) + for model, info in model_info.items(): model = model.replace("/", "--") files = glob(f"results/{model}--bigcodebench-*.jsonl") @@ -131,27 +131,21 @@ def split_gen(): model = info["link"].split("https://huggingface.co/")[-1].replace("/", "--") for file in files: + if "-sanitized" not in file or "calibrated" not in file: + continue + _, suffix = os.path.basename(file).split("--bigcodebench-") with open(file, "r") as f: data = f.readlines() - if "-sanitized" in file: - if "calibrated" in file: - if info["prompted"]: - if suffix.startswith("complete"): - with open(f"sanitized_calibrated_samples/complete/{model}--bigcodebench-{suffix}", "w") as f: - f.writelines(data) - else: - with open(f"sanitized_calibrated_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f: - f.writelines(data) + split_type = "hard" if "-hard-" in file else "full" + if info["prompted"]: + if suffix.startswith("complete") or suffix.startswith("hard-complete"): + with open(f"sanitized_calibrated_samples/{split_type}/complete/{model}--bigcodebench-{suffix}", "w") as f: + f.writelines(data) else: - if suffix.startswith("complete"): - with open(f"sanitized_samples/complete/{model}--bigcodebench-{suffix}", "w") as f: - f.writelines(data) - else: - with open(f"sanitized_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f: - f.writelines(data) - + with open(f"sanitized_calibrated_samples/{split_type}/instruct/{model}--bigcodebench-{suffix}", "w") as f: + f.writelines(data) def read_task_perf(tids, task="complete"): model_results = dict() @@ -302,7 +296,7 @@ def get_perf_df(data_dict): if __name__ == "__main__": - # split_gen() + split_gen() bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.1") bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.1") bcb_config = { From f087e3b03ce1df72cf889b201b421bd90346d445 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 2 Mar 2025 20:31:18 +0800 Subject: [PATCH 308/325] doc: add new model outputs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 94ad2ef..d3913d9 100755 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ Please make sure your HF access token has the `Make calls to inference providers ## 💻 LLM-generated Code We share pre-generated code samples from LLMs we have [evaluated](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard) on the full set: -* See the attachment of our [v0.2.1.post7](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.1.post7). We include `sanitized_samples_calibrated.zip` for your convenience. +* See the attachment of our [v0.2.4](https://github.com/bigcode-project/bigcodebench/releases/tag/v0.2.4). We include `sanitized_samples_calibrated.zip` for your convenience. ## 🧑 Advanced Usage From 6d967338737d4fa02cb2a8d19207528278282321 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 2 Mar 2025 21:42:32 +0800 Subject: [PATCH 309/325] feat: support vllm lora --- bigcodebench/generate.py | 2 ++ bigcodebench/provider/__init__.py | 3 +++ bigcodebench/provider/vllm.py | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 9823d0c..c5fa368 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -127,6 +127,7 @@ def run_codegen( split: str, subset: str, root: str = "bcb_results", + lora_path: str = None, bs: Optional[int] = None, n_samples: int = 1, temperature: float = 0.0, @@ -174,6 +175,7 @@ def run_codegen( backend=backend, subset=subset, split=split, + lora_path=lora_path, temperature=temperature, max_new_tokens=max_new_tokens, reasoning_effort=reasoning_effort, diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index f76ec29..202d049 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -6,6 +6,7 @@ def make_model( backend: str, subset: str, split: str, + lora_path: str = None, dataset: str = "bigcodebench", temperature: float = 0.0, max_new_tokens: int = 1280, @@ -38,6 +39,7 @@ def make_model( name=model, subset=subset, split=split, + lora_path=lora_path, temperature=temperature, max_new_tokens=max_new_tokens, revision=revision, @@ -58,6 +60,7 @@ def make_model( name=model, subset=subset, split=split, + lora_path=lora_path, temperature=temperature, max_new_tokens=max_new_tokens, revision=revision, diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index cc928e4..570d4c5 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -3,6 +3,8 @@ from transformers import AutoTokenizer from vllm import LLM, SamplingParams +from vllm.lora.request import LoRARequest +from huggingface_hub import snapshot_download from bigcodebench.provider.base import DecoderBase from bigcodebench.provider.utility import ( @@ -11,7 +13,7 @@ ) class VllmDecoder(DecoderBase): - def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: + def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) -> None: super().__init__(name, **kwargs) kwargs = { @@ -29,7 +31,17 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None: else: if self.prefill and "```" in self.response_prefix: self.eos += ["\n```\n"] - self.llm = LLM(model=name, max_model_len=self.max_new_tokens, **kwargs) + + self.lora_request = None + if lora_path: + local_lora_path = snapshot_download(lora_path) + self.lora_request = LoRARequest( + "lora", + 1, + local_lora_path, + ) + + self.llm = LLM(model=name, max_model_len=self.max_new_tokens, enable_lora=True if self.lora_path else False, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) def is_direct_completion(self) -> bool: @@ -64,6 +76,7 @@ def codegen( stop=self.eos, skip_special_tokens=self.skip_special_tokens, ), + lora_request=self.lora_request, use_tqdm=True, ) From 82fc40dfe33381b8bdbe5c695414afa5a543ba16 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 2 Mar 2025 21:50:48 +0800 Subject: [PATCH 310/325] fix: vllm lora attribute --- bigcodebench/provider/vllm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 570d4c5..25f00b4 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -41,7 +41,7 @@ def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) - local_lora_path, ) - self.llm = LLM(model=name, max_model_len=self.max_new_tokens, enable_lora=True if self.lora_path else False, **kwargs) + self.llm = LLM(model=name, max_model_len=self.max_new_tokens, enable_lora=True if self.lora_request else False, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) def is_direct_completion(self) -> bool: From d37847db62972decb626645699e403ed237b0d73 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Sun, 2 Mar 2025 21:57:21 +0800 Subject: [PATCH 311/325] fix: customize lora output file --- bigcodebench/generate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index c5fa368..87b67ea 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -197,9 +197,12 @@ def run_codegen( if backend == "openai" and reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): model = model + f"--{reasoning_effort}" + if lora_path: + model = model + f"--lora-{lora_path}" + if backend == "anthropic" and reasoning_budget and reasoning_beta: model = model + f"--{reasoning_budget}-{reasoning_beta}" - + if skip_prefill: identifier = model.replace("/", "--") + "--skip_prefill" + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl" else: From fa21527b1fdd727fd6f629408e16a65813231823 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 25 Mar 2025 00:02:05 +0800 Subject: [PATCH 312/325] feat: add model release date --- analysis/utils.py | 252 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 250 insertions(+), 2 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index 430e113..ec774c7 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -8,6 +8,7 @@ "act_param": 6.7, "open-data": "Partial", "reasoning": False, + "date": "2024-12-04", }, "bigcode/starcoder2-15b-instruct-v0.1": { "name": "StarCoder2-15B-Instruct-v0.1", @@ -18,6 +19,7 @@ "act_param": 15, "open-data": "Full", "reasoning": False, + "date": "2024-04-30" }, "bigcode/starcoder2-3b": { "name": "StarCoder2-3B", @@ -28,6 +30,7 @@ "act_param": 3, "open-data": "Full", "reasoning": False, + "date": "2024-02-29" }, "bigcode/starcoder2-7b": { "name": "StarCoder2-7B", @@ -38,6 +41,7 @@ "act_param": 7, "open-data": "Full", "reasoning": False, + "date": "2024-02-29" }, "bigcode/starcoder2-15b": { "name": "StarCoder2-15B", @@ -48,6 +52,7 @@ "act_param": 15, "open-data": "Full", "reasoning": False, + "date": "2024-02-29" }, "Qwen/CodeQwen1.5-7B": { "name": "CodeQwen1.5-7B", @@ -58,6 +63,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-04-16" }, "google/codegemma-2b": { "name": "CodeGemma-2B", @@ -68,6 +74,7 @@ "act_param": 2, "open-data": "None", "reasoning": False, + "date": "2024-04-10" }, "google/codegemma-7b": { "name": "CodeGemma-7B", @@ -78,6 +85,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-04-10" }, "google/codegemma-7b-it": { "name": "CodeGemma-7B-Instruct", @@ -88,6 +96,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-04-10" }, "gpt-3.5-turbo-0125": { "name": "GPT-3.5-Turbo-0125", @@ -98,6 +107,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-01-25" }, "gpt-4o": { "name": "GPT-4o-2024-05-13", @@ -108,6 +118,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-05-13" }, "gpt-4-turbo-2024-04-09": { "name": "GPT-4-Turbo-2024-04-09", @@ -118,6 +129,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-04-09" }, "gpt-4-0613": { "name": "GPT-4-0613", @@ -128,6 +140,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-06-13" }, "codellama/CodeLlama-7b-hf": { "name": "CodeLlama-7B-Base", @@ -138,6 +151,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "codellama/CodeLlama-13b-hf": { "name": "CodeLlama-13B-Base", @@ -148,6 +162,7 @@ "act_param": 13, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "codellama/CodeLlama-7b-Instruct-hf": { "name": "CodeLlama-7B-Instruct", @@ -158,6 +173,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "codellama/CodeLlama-13b-Instruct-hf": { "name": "CodeLlama-13B-Instruct", @@ -168,6 +184,7 @@ "act_param": 13, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "mistral-large-2402": { "name": "Mistral-Large-2402", @@ -178,6 +195,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-02-26" }, "mistral-small-2402": { "name": "Mistral-Small-2402", @@ -188,6 +206,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-02-26" }, "mistralai/Mixtral-8x22B-v0.1": { "name": "Mixtral-8x22B-Base", @@ -198,6 +217,7 @@ "act_param": 44, "open-data": "None", "reasoning": False, + "date": "2024-04-17" }, "mistralai/Mixtral-8x22B-Instruct-v0.1": { "name": "Mixtral-8x22B-Instruct", @@ -208,6 +228,7 @@ "act_param": 44, "open-data": "None", "reasoning": False, + "date": "2024-04-17" }, "codellama/CodeLlama-34b-hf": { "name": "CodeLlama-34B-Base", @@ -218,6 +239,7 @@ "act_param": 34, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "codellama/CodeLlama-34b-Instruct-hf": { "name": "CodeLlama-34B-Instruct", @@ -228,6 +250,7 @@ "act_param": 34, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "codellama/CodeLlama-70b-hf": { "name": "CodeLlama-70B-Base", @@ -238,6 +261,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "codellama/CodeLlama-70b-Instruct-hf": { "name": "CodeLlama-70B-Instruct", @@ -248,6 +272,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "Qwen/CodeQwen1.5-7B-Chat": { "name": "CodeQwen1.5-7B-Chat", @@ -258,6 +283,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-04-16" }, "Qwen/Qwen1.5-110B-Chat": { "name": "Qwen1.5-110B-Chat", @@ -268,6 +294,7 @@ "act_param": 110, "open-data": "None", "reasoning": False, + "date": "2024-04-26" }, "Qwen/Qwen1.5-72B-Chat": { "name": "Qwen1.5-72B-Chat", @@ -278,6 +305,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-04-26" }, "Qwen/Qwen1.5-32B-Chat": { "name": "Qwen1.5-32B-Chat", @@ -288,6 +316,7 @@ "act_param": 32, "open-data": "None", "reasoning": False, + "date": "2024-04-26" }, "deepseek-ai/DeepSeek-V2-Chat": { "name": "DeepSeek-V2-Chat", @@ -298,6 +327,7 @@ "act_param": 21, "open-data": "None", "reasoning": False, + "date": "2024-04-06" }, "deepseek-ai/deepseek-coder-1.3b-base": { "name": "DeepSeek-Coder-1.3B-Base", @@ -308,6 +338,7 @@ "act_param": 1.3, "open-data": "None", "reasoning": False, + "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-1.3b-instruct": { "name": "DeepSeek-Coder-1.3B-Instruct", @@ -318,6 +349,7 @@ "act_param": 1.3, "open-data": "None", "reasoning": False, + "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-33b-base": { "name": "DeepSeek-Coder-33B-Base", @@ -328,6 +360,7 @@ "act_param": 33, "open-data": "None", "reasoning": False, + "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-33b-instruct": { "name": "DeepSeek-Coder-33B-Instruct", @@ -338,6 +371,7 @@ "act_param": 33, "open-data": "None", "reasoning": False, + "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-6.7b-base": { "name": "DeepSeek-Coder-6.7B-Base", @@ -348,6 +382,7 @@ "act_param": 6.7, "open-data": "None", "reasoning": False, + "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-6.7b-instruct": { "name": "DeepSeek-Coder-6.7B-Instruct", @@ -358,6 +393,7 @@ "act_param": 6.7, "open-data": "None", "reasoning": False, + "date": "2023-10-28" }, "meta-llama/Meta-Llama-3-70B": { "name": "Llama-3-70B-Base", @@ -368,6 +404,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-04-18" }, "meta-llama/Meta-Llama-3-70B-Instruct": { "name": "Llama-3-70B-Instruct", @@ -378,6 +415,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-04-18" }, "meta-llama/Meta-Llama-3-8B": { "name": "Llama-3-8B-Base", @@ -388,6 +426,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-04-18" }, "meta-llama/Meta-Llama-3-8B-Instruct": { "name": "Llama-3-8B-Instruct", @@ -398,6 +437,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-04-18" }, "ibm-granite/granite-3b-code-instruct": { "name": "Granite-Code-3B-Instruct", @@ -408,6 +448,7 @@ "act_param": 3, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "ibm-granite/granite-8b-code-instruct": { "name": "Granite-Code-8B-Instruct", @@ -418,6 +459,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "ibm-granite/granite-20b-code-instruct": { "name": "Granite-Code-20B-Instruct", @@ -428,6 +470,7 @@ "act_param": 20, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "ibm-granite/granite-34b-code-instruct": { "name": "Granite-Code-34B-Instruct", @@ -438,6 +481,7 @@ "act_param": 34, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "ibm-granite/granite-3b-code-base": { "name": "Granite-Code-3B-Base", @@ -448,6 +492,7 @@ "act_param": 3, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "ibm-granite/granite-8b-code-base": { "name": "Granite-Code-8B-Base", @@ -458,6 +503,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "ibm-granite/granite-20b-code-base": { "name": "Granite-Code-20B-Base", @@ -468,6 +514,7 @@ "act_param": 20, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "ibm-granite/granite-34b-code-base": { "name": "Granite-Code-34B-Base", @@ -478,6 +525,7 @@ "act_param": 34, "open-data": "None", "reasoning": False, + "date": "2024-05-06" }, "claude-3-haiku-20240307": { "name": "Claude-3-Haiku-20240307", @@ -488,6 +536,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-03-07" }, "claude-3-sonnet-20240229": { "name": "Claude-3-Sonnet-20240229", @@ -498,6 +547,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-02-29" }, "claude-3-opus-20240229": { "name": "Claude-3-Opus-20240229", @@ -508,6 +558,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-02-29" }, "01-ai/Yi-1.5-34B-Chat": { "name": "Yi-1.5-34B-Chat", @@ -518,6 +569,7 @@ "act_param": 34, "open-data": "None", "reasoning": False, + "date": "2024-05-20" }, "01-ai/Yi-1.5-34B": { "name": "Yi-1.5-34B", @@ -528,6 +580,7 @@ "act_param": 34, "open-data": "None", "reasoning": False, + "date": "2024-05-20" }, "01-ai/Yi-1.5-9B-Chat": { "name": "Yi-1.5-9B-Chat", @@ -538,6 +591,7 @@ "act_param": 9, "open-data": "None", "reasoning": False, + "date": "2024-05-20" }, "01-ai/Yi-1.5-9B": { "name": "Yi-1.5-9B", @@ -548,6 +602,7 @@ "act_param": 9, "open-data": "None", "reasoning": False, + "date": "2024-05-20" }, "01-ai/Yi-1.5-6B-Chat": { "name": "Yi-1.5-6B-Chat", @@ -558,6 +613,7 @@ "act_param": 6, "open-data": "None", "reasoning": False, + "date": "2024-05-20" }, "01-ai/Yi-1.5-6B": { "name": "Yi-1.5-6B", @@ -568,6 +624,7 @@ "act_param": 6, "open-data": "None", "reasoning": False, + "date": "2024-05-20" }, "Qwen/Qwen2-57B-A14B": { "name": "Qwen2-57B-A14B", @@ -578,6 +635,7 @@ "act_param": 14, "open-data": "None", "reasoning": False, + "date": "2024-06-07" }, "Qwen/Qwen2-7B-Instruct": { "name": "Qwen2-7B-Instruct", @@ -588,6 +646,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-06-07" }, "Qwen/Qwen2-72B-Chat": { "name": "Qwen2-72B-Chat", @@ -598,6 +657,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-06-07" }, "gemini-1.5-pro": { "name": "Gemini-1.5-Pro-API-0514", @@ -608,6 +668,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-05-14" }, "gemini-1.5-flash": { "name": "Gemini-1.5-Flash-API-0514", @@ -618,6 +679,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-05-14" }, "m-a-p/OpenCodeInterpreter-DS-33B": { "name": "OpenCodeInterpreter-DS-33B", @@ -628,6 +690,7 @@ "act_param": 33, "open-data": "Partial", "reasoning": False, + "date": "2024-02-22" }, "m-a-p/OpenCodeInterpreter-DS-6.7B": { "name": "OpenCodeInterpreter-DS-6.7B", @@ -638,6 +701,7 @@ "act_param": 6.7, "open-data": "Partial", "reasoning": False, + "date": "2024-02-22" }, "m-a-p/OpenCodeInterpreter-DS-1.3B": { "name": "OpenCodeInterpreter-DS-1.3B", @@ -648,6 +712,7 @@ "act_param": 1.3, "open-data": "Partial", "reasoning": False, + "date": "2024-02-22" }, "microsoft/Phi-3-medium-128k-instruct": { "name": "Phi-3-Medium-128K-Instruct", @@ -658,6 +723,7 @@ "act_param": 14, "open-data": "None", "reasoning": False, + "date": "2024-05-21" }, "microsoft/Phi-3-small-128k-instruct": { "name": "Phi-3-Small-128K-Instruct", @@ -668,6 +734,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-05-21" }, "codestral-2405": { "name": "Codestral-22B-v0.1", @@ -678,6 +745,7 @@ "act_param": 22, "open-data": "None", "reasoning": False, + "date": "2024-05-23" }, "codestral-mamba-2407": { "name": "Codestral-Mamba", @@ -688,6 +756,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-07-16" }, "mistralai/Mistral-7B-Instruct-v0.3": { "name": "Mistral-7B-Instruct-v0.3", @@ -698,6 +767,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-05-22" }, "mistralai/Mistral-7B-v0.3": { "name": "Mistral-7B-v0.3", @@ -708,6 +778,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-05-22" }, "CohereForAI/c4ai-command-r-plus": { "name": "Command R+", @@ -718,6 +789,7 @@ "act_param": 104, "open-data": "None", "reasoning": False, + "date": "2024-04-04" }, "deepseek-coder": { "name": "DeepSeek-Coder-V2-Instruct", @@ -728,6 +800,7 @@ "act_param": 21, "open-data": "None", "reasoning": True, + "date": "2024-06-17" }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": { "name": "DeepSeek-Coder-V2-Lite-Instruct", @@ -738,6 +811,7 @@ "act_param": 2.4, "open-data": "None", "reasoning": False, + "date": "2024-06-17" }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Base": { "name": "DeepSeek-Coder-V2-Lite-Base", @@ -748,6 +822,7 @@ "act_param": 2.4, "open-data": "None", "reasoning": False, + "date": "2024-06-17" }, "claude-3-5-sonnet-20240620": { "name": "Claude-3.5-Sonnet-20240620", @@ -758,6 +833,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-06-20" }, "NousResearch/Hermes-2-Theta-Llama-3-70B": { "name": "Hermes-2-Theta-Llama-3-70B", @@ -768,6 +844,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-06-24" }, "microsoft/wavecoder-ultra-6.7b": { "name": "WaveCoder-Ultra-6.7B", @@ -778,6 +855,7 @@ "act_param": 6.7, "open-data": "None", "reasoning": False, + "date": "2023-12-26" }, "google/gemma-2-9b-it": { "name": "Gemma-2-9B-Instruct", @@ -788,6 +866,7 @@ "act_param": 9, "open-data": "None", "reasoning": False, + "date": "2024-06-19" }, "Bin12345/AutoCoder": { "name": "AutoCoder", @@ -798,6 +877,7 @@ "act_param": 33, "open-data": "None", "reasoning": False, + "date": "2024-05-23" }, "Bin12345/AutoCoder_S_6.7B": { "name": "AutoCoder-S-6.7B", @@ -808,6 +888,7 @@ "act_param": 6.7, "open-data": "None", "reasoning": False, + "date": "2024-05-23" }, "Bin12345/AutoCoder_QW_7B": { "name": "AutoCoder-QW-7B", @@ -818,6 +899,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-05-23" }, "SenseLLM/ReflectionCoder-DS-33B": { "name": "ReflectionCoder-DS-33B", @@ -828,6 +910,7 @@ "act_param": 33, "open-data": "Partial", "reasoning": False, + "date": "2024-05-27" }, "SenseLLM/ReflectionCoder-DS-6.7B": { "name": "ReflectionCoder-DS-6.7B", @@ -838,6 +921,7 @@ "act_param": 6.7, "open-data": "Partial", "reasoning": False, + "date": "2024-05-27" }, "SenseLLM/ReflectionCoder-CL-34B": { "name": "ReflectionCoder-CL-34B", @@ -848,6 +932,7 @@ "act_param": 34, "open-data": "Partial", "reasoning": False, + "date": "2024-05-27" }, "SenseLLM/ReflectionCoder-CL-7B": { "name": "ReflectionCoder-CL-7B", @@ -858,6 +943,7 @@ "act_param": 7, "open-data": "Partial", "reasoning": False, + "date": "2024-05-27" }, "new-microsoft/Phi-3-mini-128k-instruct": { "name": "Phi-3.1-Mini-128K-Instruct", @@ -868,6 +954,7 @@ "act_param": 3.8, "open-data": "None", "reasoning": False, + "date": "2024-05-21" }, "old-microsoft/Phi-3-mini-128k-instruct": { "name": "Phi-3-Mini-128K-Instruct", @@ -878,6 +965,7 @@ "act_param": 3.8, "open-data": "None", "reasoning": False, + "date": "2024-05-21" }, "internlm/internlm2_5-7b-chat": { "name": "InternLM2.5-7B-Chat", @@ -888,6 +976,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-07-03" }, "NousResearch/Hermes-2-Pro-Llama-3-70B": { "name": "Hermes-2-Pro-Llama-3-70B", @@ -898,6 +987,7 @@ "act_param": 70, "open-data": "Partial", "reasoning": False, + "date": "2024-06-27" }, "new-deepseek-chat": { "name": "DeepSeek-V2-Chat (2024-06-28)", @@ -908,6 +998,7 @@ "act_param": 21, "open-data": "None", "reasoning": True, + "date": "2024-06-28" }, "vllm-google/gemma-2-27b-it": { "name": "Gemma-2-27B-Instruct", @@ -918,6 +1009,7 @@ "act_param": 27, "open-data": "None", "reasoning": False, + "date": "2024-06-19" }, "Artigenz/Artigenz-Coder-DS-6.7B": { "name": "Artigenz-Coder-DS-6.7B", @@ -928,6 +1020,7 @@ "act_param": 6.7, "open-data": "None", "reasoning": False, + "date": "2024-04-16" }, "openchat/openchat-3.6-8b-20240522": { "name": "OpenChat-3.6-8B-20240522", @@ -938,6 +1031,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-05-22" }, "Phind/Phind-CodeLlama-34B-v2": { "name": "Phind-CodeLlama-34B-v2", @@ -948,6 +1042,7 @@ "act_param": 34, "open-data": "None", "reasoning": False, + "date": "2023-08-25" }, "yi-large": { "name": "Yi-Large", @@ -958,6 +1053,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-05-13" }, "THUDM/codegeex4-all-9b": { "name": "CodeGeex4-All-9B", @@ -968,6 +1064,7 @@ "act_param": 9, "open-data": "None", "reasoning": False, + "date": "2024-07-05" }, "gpt-4o-mini-2024-07-18": { "name": "GPT-4o-mini-2024-07-18", @@ -978,6 +1075,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-07-18" }, "Nexusflow/Athene-70B": { "name": "Athene-70B", @@ -988,6 +1086,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-07-20" }, "NTQAI/Nxcode-CQ-7B-orpo": { "name": "Nxcode-CQ-7B-Orpo", @@ -998,6 +1097,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-04-25" }, "migtissera/Llama-3-70B-Synthia-v3.5": { "name": "Llama-3-70B-Synthia-v3.5", @@ -1008,6 +1108,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-05-27" }, "migtissera/Tess-v2.5.2-Qwen2-72B": { "name": "Tess-v2.5.2-Qwen2-72B", @@ -1018,6 +1119,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-07-18" }, "WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5": { "name": "WhiteRabbitNeo-33B-v1.5", @@ -1028,6 +1130,7 @@ "act_param": 33, "open-data": "None", "reasoning": False, + "date": "2024-02-10" }, "mistral-large-2407": { "name": "Mistral-Large-Instruct-2407", @@ -1038,6 +1141,7 @@ "act_param": 123, "open-data": "None", "reasoning": True, + "date": "2024-07-24" }, "meta-llama/Meta-Llama-3.1-8B-Instruct": { "name": "Llama-3.1-8B-Instruct", @@ -1048,6 +1152,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-07-23" }, "meta-llama/Meta-Llama-3.1-70B-Instruct": { "name": "Llama-3.1-70B-Instruct", @@ -1058,6 +1163,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-07-23" }, "meta--llama-3.1-405b-instruct": { "name": "Llama-3.1-405B-Instruct", @@ -1068,6 +1174,7 @@ "act_param": 405, "open-data": "None", "reasoning": False, + "date": "2024-07-23" }, "deepseek-coder-20240724": { "name": "DeepSeek-Coder-V2-Instruct (2024-07-24)", @@ -1078,6 +1185,7 @@ "act_param": 21, "open-data": "None", "reasoning": False, + "date": "2024-07-24" }, "microsoft/Phi-3.5-mini-instruct": { "name": "Phi-3.5-Mini-Instruct", @@ -1088,6 +1196,7 @@ "act_param": 3.8, "open-data": "None", "reasoning": False, + "date": "2024-04-23" }, "nv-mistralai--mistral-nemo-12b-instruct": { "name": "Mistral-Nemo-12B-Instruct", @@ -1098,6 +1207,7 @@ "act_param": 12, "open-data": "None", "reasoning": False, + "date": "2024-07-18" }, "wyt2000/InverseCoder-CL-13B": { "name": "InverseCoder-CL-13B", @@ -1108,6 +1218,7 @@ "act_param": 13, "open-data": "Partial", "reasoning": False, + "date": "2024-07-08" }, "wyt2000/InverseCoder-CL-7B": { "name": "InverseCoder-CL-7B", @@ -1118,6 +1229,7 @@ "act_param": 7, "open-data": "Partial", "reasoning": False, + "date": "2024-07-08" }, "wyt2000/InverseCoder-DS-6.7B": { "name": "InverseCoder-DS-6.7B", @@ -1128,6 +1240,7 @@ "act_param": 6.7, "open-data": "Partial", "reasoning": False, + "date": "2024-07-08" }, "gemini-1.5-pro-exp-0801": { "name": "Gemini-1.5-Pro-Exp-0801", @@ -1138,6 +1251,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-08-01" }, "gpt-4o-2024-08-06": { "name": "GPT-4o-2024-08-06", @@ -1148,6 +1262,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-08-06" }, "abacusai/Dracarys-Llama-3.1-70B-Instruct": { "name": "Dracarys-Llama-3.1-70B-Instruct", @@ -1158,6 +1273,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-08-23" }, "abacusai/Dracarys-72B-Instruct": { "name": "Dracarys-72B-Instruct", @@ -1168,6 +1284,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-08-23" }, "gemini-1.5-pro-exp-0827": { "name": "Gemini-1.5-Pro-Exp-0827", @@ -1178,6 +1295,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-08-27" }, "gemini-1.5-flash-exp-0827": { "name": "Gemini-1.5-Flash-Exp-0827", @@ -1188,6 +1306,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-08-27" }, "microsoft/Phi-3.5-mini-instruct": { "name": "Phi-3.5-Mini-Instruct", @@ -1198,6 +1317,7 @@ "act_param": 3.8, "open-data": "None", "reasoning": False, + "date": "2024-04-23" }, "abacusai/Dracarys-Llama-3.1-70B-Instruct": { "name": "Dracarys-Llama-3.1-70B-Instruct", @@ -1208,6 +1328,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-04-23" }, "abacusai/Dracarys-72B-Instruct": { "name": "Dracarys-72B-Instruct", @@ -1218,6 +1339,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-04-23" }, "deepseek-coder-v2.5": { "name": "DeepSeek-V2.5", @@ -1228,6 +1350,7 @@ "act_param": 21, "open-data": "None", "reasoning": False, + "date": "2024-09-18" }, "CohereForAI/c4ai-command-r-08-2024": { "name": "C4AI-Command-R-08-2024", @@ -1238,6 +1361,7 @@ "act_param": 32.3, "open-data": "None", "reasoning": False, + "date": "2024-08-30" }, "CohereForAI/c4ai-command-r-plus-08-2024": { "name": "C4AI-Command-R-Plus-08-2024", @@ -1248,6 +1372,7 @@ "act_param": 104, "open-data": "None", "reasoning": False, + "date": "2024-08-30" }, "ayueei--yue-coder-9b-preview": { "name": "Yi-Coder-9B-Chat", @@ -1258,6 +1383,7 @@ "act_param": 9, "open-data": "None", "reasoning": False, + "date": "2024-09-04" }, # "mattshumer/ref_70_e3_prefill": { # "name": "Reflection-Llama-3.1-70B", @@ -1286,6 +1412,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-09-12" }, "o1-mini-2024-09-12": { "name": "o1-Mini-2024-09-12 (temperature=1)", @@ -1296,6 +1423,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-09-12" }, "Qwen/Qwen2.5-Coder-1.5B-Instruct": { "name": "Qwen2.5-Coder-1.5B-Instruct", @@ -1306,6 +1434,7 @@ "act_param": 1.5, "open-data": "None", "reasoning": False, + "date": "2024-11-12" }, "Qwen/Qwen2.5-Coder-7B-Instruct": { "name": "Qwen2.5-Coder-7B-Instruct", @@ -1316,6 +1445,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-11-12" }, "gemini-1.5-pro-002": { "name": "Gemini-1.5-Pro-002", @@ -1326,6 +1456,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-09-25" }, "mistralai/Mistral-Small-Instruct-2409": { "name": "Mistral-Small-Instruct-2409", @@ -1336,6 +1467,7 @@ "act_param": 22.2, "open-data": "None", "reasoning": False, + "date": "2024-09-18" }, "Qwen/Qwen2.5-0.5B-Instruct": { "name": "Qwen2.5-0.5B-Instruct", @@ -1346,6 +1478,7 @@ "act_param": 0.5, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "Qwen/Qwen2.5-1.5B-Instruct": { "name": "Qwen2.5-1.5B-Instruct", @@ -1356,6 +1489,7 @@ "act_param": 1.5, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "Qwen/Qwen2.5-7B-Instruct": { "name": "Qwen2.5-7B-Instruct", @@ -1366,6 +1500,7 @@ "act_param": 7, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "Qwen/Qwen2.5-14B-Instruct": { "name": "Qwen2.5-14B-Instruct", @@ -1376,6 +1511,7 @@ "act_param": 14, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "Qwen/Qwen2.5-32B-Instruct": { "name": "Qwen2.5-32B-Instruct", @@ -1386,6 +1522,7 @@ "act_param": 32, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "Qwen/Qwen2.5-72B-Instruct": { "name": "Qwen2.5-72B-Instruct", @@ -1396,6 +1533,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "meta-llama/Llama-3.2-1B-Instruct": { "name": "Llama-3.2-1B-Instruct", @@ -1406,6 +1544,7 @@ "act_param": 1, "open-data": "None", "reasoning": False, + "date": "2024-09-25" }, "meta-llama/Llama-3.2-3B-Instruct": { "name": "Llama-3.2-3B-Instruct", @@ -1416,6 +1555,7 @@ "act_param": 3, "open-data": "None", "reasoning": False, + "date": "2024-09-25" }, "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { "name": "Llama-3.1-Nemotron-70B-Instruct", @@ -1426,6 +1566,7 @@ "act_param": 70, "open-data": "Partial", "reasoning": False, + "date": "2024-09-25" }, "claude-3-5-sonnet-20241022": { "name": "Claude-3.5-Sonnet-20241022", @@ -1436,6 +1577,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-10-22" }, "ibm-granite/granite-3.0-8b-instruct": { "name": "Granite-3.0-8B-Instruct", @@ -1446,6 +1588,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-10-21" }, "ibm-granite/granite-3.0-2b-instruct": { "name": "Granite-3.0-2B-Instruct", @@ -1456,6 +1599,7 @@ "act_param": 2, "open-data": "None", "reasoning": False, + "date": "2024-10-21" }, "grok-beta--main": { "name": "Grok-Beta", @@ -1466,6 +1610,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-03-17" }, "claude-3-5-haiku-20241022--main": { "name": "Claude-3.5-Haiku-20241022", @@ -1476,6 +1621,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-10-22" }, "Qwen/Qwen2.5-Coder-14B-Instruct--main": { "name": "Qwen2.5-Coder-14B-Instruct", @@ -1486,6 +1632,7 @@ "act_param": 14, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "Qwen/Qwen2.5-Coder-32B-Instruct--main": { "name": "Qwen2.5-Coder-32B-Instruct", @@ -1496,6 +1643,7 @@ "act_param": 32, "open-data": "None", "reasoning": False, + "date": "2024-09-19" }, "infly/OpenCoder-1.5B-Instruct--main": { "name": "OpenCoder-1.5B-Instruct", @@ -1506,6 +1654,7 @@ "act_param": 1.5, "open-data": "None", "reasoning": False, + "date": "2024-11-09" }, "infly/OpenCoder-8B-Instruct--main": { "name": "OpenCoder-8B-Instruct", @@ -1516,6 +1665,7 @@ "act_param": 8, "open-data": "None", "reasoning": False, + "date": "2024-11-09" }, "microsoft/Phi-3.5-mini-instruct--main": { "name": "Phi-3.5-Mini-Instruct", @@ -1526,6 +1676,7 @@ "act_param": 3.8, "open-data": "None", "reasoning": False, + "date": "2024-08-21" }, "Nexusflow/Athene-V2-Agent--main": { "name": "Athene-V2-Agent", @@ -1536,6 +1687,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-11-14" }, "Nexusflow/Athene-V2-Chat--main": { "name": "Athene-V2-Chat", @@ -1546,6 +1698,7 @@ "act_param": 72, "open-data": "None", "reasoning": False, + "date": "2024-11-14" }, "gemini-exp-1114--main": { "name": "Gemini-Exp-1114", @@ -1556,6 +1709,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-11-14" }, "gpt-4o-2024-11-20--main": { "name": "GPT-4o-2024-11-20", @@ -1566,6 +1720,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-11-20" }, "gemini-exp-1121--main": { "name": "Gemini-Exp-1121", @@ -1576,6 +1731,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-11-21" }, "gemini-exp-1206--main": { "name": "Gemini-Exp-1206", @@ -1586,6 +1742,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-12-06" }, "meta-llama--Llama-3.3-70B-Instruct--main": { "name": "Llama-3.3-70B-Instruct", @@ -1596,6 +1753,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2024-12-19" }, "deepseek-ai--DeepSeek-V2.5-1210--main": { "name": "DeepSeek-V2.5-1210", @@ -1606,6 +1764,7 @@ "act_param": 21, "open-data": "None", "reasoning": False, + "date": "2024-12-10" }, "gemini-2.0-flash-exp--main": { "name": "Gemini-2.0-Flash-Exp", @@ -1616,6 +1775,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-12-11" }, "gemini-2.0-flash-thinking-exp-1219--main": { "name": "Gemini-2.0-Flash-Thinking-Exp-1219", @@ -1626,6 +1786,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2024-12-19" }, "gemini-2.0-flash-thinking-exp-01-21--main": { "name": "Gemini-2.0-Flash-Thinking-Exp-01-21", @@ -1636,6 +1797,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2025-01-21" }, "o1-2024-12-17--main": { "name": "o1-2024-12-17 (temperature=1, reasoning=medium)", @@ -1646,6 +1808,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-12-17" }, "o1-2024-12-17--low--main": { "name": "o1-2024-12-17 (temperature=1, reasoning=low)", @@ -1656,6 +1819,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-12-17" }, "o1-2024-12-17--high--main": { "name": "o1-2024-12-17 (temperature=1, reasoning=high)", @@ -1666,16 +1830,18 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2024-12-17" }, "deepseek-v3-chat--main": { - "name": "DeepSeek-V3-Chat", - "link": "https://huggingface.co/deepseek-ai/DeepSeek-V3-Chat", + "name": "DeepSeek-V3", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-V3", "prompted": True, "moe": True, "size": 671, "act_param": 37, "open-data": "None", "reasoning": True, + "date": "2024-12-26" }, "microsoft--phi-4--main": { "name": "Phi-4", @@ -1686,6 +1852,7 @@ "act_param": 14.7, "open-data": "None", "reasoning": False, + "date": "2024-12-13" }, "deepseek-reasoner--main": { "name": "DeepSeek-R1", @@ -1696,6 +1863,7 @@ "act_param": 37, "open-data": "None", "reasoning": True, + "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Llama-70B--main": { "name": "DeepSeek-R1-Distill-Llama-70B", @@ -1706,6 +1874,7 @@ "act_param": 70, "open-data": "None", "reasoning": False, + "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B--main": { "name": "DeepSeek-R1-Distill-Qwen-32B", @@ -1716,6 +1885,7 @@ "act_param": 32, "open-data": "None", "reasoning": False, + "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B--main": { "name": "DeepSeek-R1-Distill-Qwen-14B", @@ -1726,6 +1896,7 @@ "act_param": 14, "open-data": "None", "reasoning": False, + "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Llama-8B--main": { "name": "DeepSeek-R1-Distill-Llama-8B", @@ -1736,6 +1907,7 @@ "act_param": 14, "open-data": "None", "reasoning": False, + "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B--main": { "name": "DeepSeek-R1-Distill-Qwen-7B", @@ -1746,6 +1918,7 @@ "act_param": 14, "open-data": "None", "reasoning": False, + "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B--main": { "name": "DeepSeek-R1-Distill-Qwen-1.5B", @@ -1756,6 +1929,7 @@ "act_param": 1.5, "open-data": "None", "reasoning": False, + "date": "2025-01-20" }, "mistralai/Mistral-Small-24B-Instruct-2501--main": { "name": "Mistral-Small-24B-Instruct-2501", @@ -1766,6 +1940,7 @@ "act_param": 24, "open-data": "None", "reasoning": False, + "date": "2025-01-31" }, "o3-mini-2025-01-31--medium--main": { "name": "o3-mini-2025-01-31 (temperature=1, reasoning=medium)", @@ -1776,6 +1951,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2025-01-31" }, "o3-mini-2025-01-31--low--main": { "name": "o3-mini-2025-01-31 (temperature=1, reasoning=low)", @@ -1786,6 +1962,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2025-01-31" }, "o3-mini-2025-01-31--high--main": { "name": "o3-mini-2025-01-31 (temperature=1, reasoning=high)", @@ -1796,6 +1973,7 @@ "act_param": None, "open-data": "None", "reasoning": True, + "date": "2025-01-31" }, "gemini-2.0-flash-001--main": { "name": "Gemini-2.0-Flash-001", @@ -1806,6 +1984,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2025-02-05" }, "gemini-2.0-flash-exp--main": { "name": "Gemini-2.0-Flash-Exp", @@ -1816,6 +1995,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2025-02-05" }, "gemini-2.0-flash-lite-preview-02-05--main": { "name": "Gemini-2.0-Flash-Lite-Preview-02-05", @@ -1826,6 +2006,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2025-02-05" }, "gemini-2.0-pro-exp-02-05--main": { "name": "Gemini-2.0-Pro-Exp-02-05", @@ -1836,6 +2017,7 @@ "act_param": None, "open-data": "None", "reasoning": False, + "date": "2025-02-05" }, "NovaSky-AI--Sky-T1-32B-Flash--main": { "name": "Sky-T1-32B-Flash", @@ -1846,6 +2028,7 @@ "act_param": 32, "open-data": "None", "reasoning": False, + "date": "2025-01-12" }, "NovaSky-AI--Sky-T1-32B-Preview--main": { "name": "Sky-T1-32B-Preview", @@ -1856,6 +2039,7 @@ "act_param": 32, "open-data": "None", "reasoning": False, + "date": "2025-01-12" }, "Qwen--QwQ-32B-Preview--main": { "name": "QwQ-32B-Preview", @@ -1866,5 +2050,69 @@ "act_param": 32, "open-data": "None", "reasoning": False, + "date": "2024-11-28" + }, + "claude-3-7-sonnet-20250219--main": { + "name": "Claude-3-Haiku-20240307", + "link": "https://www.anthropic.com/news/claude-3-family", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": True, + "date": "2025-02-19" + }, + "chatgpt-4o-latest--main": { + "name": "ChatGPT-4o-latest-20250129", + "link": "https://chat.openai.com/", + "open-data": "None", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "date": "2025-01-29" + }, + "Kwaipilot--KwaiCoder-23B-A4B-v1--main": { + "name": "KwaiCoder-23B-A4B-v1", + "link": "https://huggingface.co/Kwaipilot/KwaiCoder-23B-A4B-v1", + "open-data": "None", + "prompted": False, + "moe": True, + "size": 23, + "act_param": 4, + "date": "2025-01-25" + }, + "qwen-max-latest--main": { + "name": "Qwen2.5-Max", + "link": "https://qwenlm.github.io/blog/qwen2.5-max/", + "open-data": "None", + "prompted": True, + "moe": True, + "size": None, + "act_param": None, + "date": "2025-01-28" + }, + "claude-3-7-sonnet-20250219--3200-output-128k-2025-02-19--main": { + "name": "Claude-3.7-Sonnet-20250219 (temperature=1, length=12800, reasoning=3200)", + "link": "https://www.anthropic.com/news/claude-3-7-sonnet", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": True, + "date": "2025-02-19" + }, + "claude-3-7-sonnet-20250219--main": { + "name": "Claude-3.7-Sonnet-20250219", + "link": "https://www.anthropic.com/news/claude-3-7-sonnet", + "prompted": True, + "moe": False, + "size": None, + "act_param": None, + "open-data": "None", + "reasoning": True, + "date": "2025-02-19" }, } From 5f0743d0a6874fd6fdfe6ab616fe7f65145fb038 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 25 Mar 2025 00:03:20 +0800 Subject: [PATCH 313/325] fix: remove vllm max length --- bigcodebench/provider/vllm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 25f00b4..60b2285 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -41,7 +41,7 @@ def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) - local_lora_path, ) - self.llm = LLM(model=name, max_model_len=self.max_new_tokens, enable_lora=True if self.lora_request else False, **kwargs) + self.llm = LLM(model=name, enable_lora=True if self.lora_request else False, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) def is_direct_completion(self) -> bool: From 3513d997f55c383dec3436d7b43704a4affbc8d9 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 25 Mar 2025 00:13:39 +0800 Subject: [PATCH 314/325] fix: hardcode the model max length for vllm --- bigcodebench/provider/vllm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 60b2285..229e4c9 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -41,7 +41,8 @@ def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) - local_lora_path, ) - self.llm = LLM(model=name, enable_lora=True if self.lora_request else False, **kwargs) + # max_model_len is set to max_new_tokens * 10 + self.llm = LLM(model=name, max_model_len=self.max_new_tokens * 10, enable_lora=True if self.lora_request else False, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) def is_direct_completion(self) -> bool: From 00fc9bb98c932424c2e9bf82ab417142aaca5e1d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 25 Mar 2025 02:20:37 +0800 Subject: [PATCH 315/325] fix model metadata --- analysis/utils.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index ec774c7..29a1cb7 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -1903,8 +1903,8 @@ "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B", "prompted": True, "moe": False, - "size": 14, - "act_param": 14, + "size": 8, + "act_param": 8, "open-data": "None", "reasoning": False, "date": "2025-01-20" @@ -1914,8 +1914,8 @@ "link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "prompted": True, "moe": False, - "size": 14, - "act_param": 14, + "size": 7, + "act_param": 7, "open-data": "None", "reasoning": False, "date": "2025-01-20" @@ -2115,4 +2115,36 @@ "reasoning": True, "date": "2025-02-19" }, + "WarriorCoder-6.7B--main": { + "name": "WarriorCoder-6.7B (Reproduced)", + "link": "https://arxiv.org/abs/2412.17395", + "open-data": "None", + "prompted": True, + "moe": False, + "date": "2025-02-18" + }, + "google--gemma-3-27b-it--main": { + "name": "Gemma-3-27B-Instruct", + "link": "https://huggingface.co/google/gemma-3-27b-it", + "open-data": "None", + "prompted": True, + "moe": False, + "date": "2025-03-12" + }, + "Qwen--QwQ-32B--skip_prefill--main": { + "name": "QwQ-32B (w/ Reasoning)", + "link": "https://huggingface.co/Qwen/QwQ-32B", + "open-data": "None", + "prompted": True, + "moe": False, + "date": "2025-03-06" + }, + "deepseek-chat-0324--main": { + "name": "DeepSeek-V3-0324", + "link": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324", + "open-data": "None", + "prompted": True, + "moe": False, + "date": "2025-03-24" + } } From 720681b8ecbcabbfafa6f4c1aae1ca8365d726c4 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 25 Mar 2025 02:22:11 +0800 Subject: [PATCH 316/325] feat: add max_model_len for vllm --- ADVANCED_USAGE.md | 1 + bigcodebench/generate.py | 3 +++ bigcodebench/provider/__init__.py | 2 ++ bigcodebench/provider/vllm.py | 5 ++--- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ADVANCED_USAGE.md b/ADVANCED_USAGE.md index 9bb81b8..c0905ba 100755 --- a/ADVANCED_USAGE.md +++ b/ADVANCED_USAGE.md @@ -50,6 +50,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio - `--n_samples`: The number of samples, default to `1` - `--temperature`: The temperature, default to `0.0` - `--max_new_tokens`: The length of max new tokens, default to `1280` +- `--max_model_len`: The length of max tokens in VLLM, default to `12800` - `--greedy`: Whether to use greedy decoding, default to `False` - `--strip_newlines`: Whether to strip newlines, default to `False`, set to `True` to strip newlines for some model series like StarCoder2 - `--direct_completion`: Whether to use direct completion, default to `False` diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 87b67ea..912abcd 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -132,6 +132,8 @@ def run_codegen( n_samples: int = 1, temperature: float = 0.0, max_new_tokens: int = 1280, + # vllm + max_model_len: int = 12800, greedy: bool = False, # openai reasoning_effort: str = "medium", @@ -178,6 +180,7 @@ def run_codegen( lora_path=lora_path, temperature=temperature, max_new_tokens=max_new_tokens, + max_model_len=max_model_len, reasoning_effort=reasoning_effort, reasoning_budget=reasoning_budget, reasoning_beta=reasoning_beta, diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py index 202d049..4cb3410 100644 --- a/bigcodebench/provider/__init__.py +++ b/bigcodebench/provider/__init__.py @@ -10,6 +10,7 @@ def make_model( dataset: str = "bigcodebench", temperature: float = 0.0, max_new_tokens: int = 1280, + max_model_len: int = 12800, # openai only reasoning_effort: str = "medium", # anthropic only @@ -42,6 +43,7 @@ def make_model( lora_path=lora_path, temperature=temperature, max_new_tokens=max_new_tokens, + max_model_len=max_model_len, revision=revision, dataset=dataset, direct_completion=direct_completion, diff --git a/bigcodebench/provider/vllm.py b/bigcodebench/provider/vllm.py index 229e4c9..41cd251 100644 --- a/bigcodebench/provider/vllm.py +++ b/bigcodebench/provider/vllm.py @@ -13,7 +13,7 @@ ) class VllmDecoder(DecoderBase): - def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) -> None: + def __init__(self, name: str, lora_path: str, dataset: str, tp: int, max_model_len: int, **kwargs) -> None: super().__init__(name, **kwargs) kwargs = { @@ -41,8 +41,7 @@ def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) - local_lora_path, ) - # max_model_len is set to max_new_tokens * 10 - self.llm = LLM(model=name, max_model_len=self.max_new_tokens * 10, enable_lora=True if self.lora_request else False, **kwargs) + self.llm = LLM(model=name, max_model_len=max_model_len, enable_lora=True if self.lora_request else False, **kwargs) self.llm.set_tokenizer(tokenizer=self.tokenizer) def is_direct_completion(self) -> bool: From c9e2cbba6618bec6ced0aa08892e4a7446d128ee Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 1 Apr 2025 01:10:09 +0800 Subject: [PATCH 317/325] update model metadata --- analysis/utils.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/analysis/utils.py b/analysis/utils.py index 29a1cb7..798499b 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -2071,6 +2071,7 @@ "moe": False, "size": None, "act_param": None, + "reasoning": True, "date": "2025-01-29" }, "Kwaipilot--KwaiCoder-23B-A4B-v1--main": { @@ -2081,6 +2082,7 @@ "moe": True, "size": 23, "act_param": 4, + "reasoning": False, "date": "2025-01-25" }, "qwen-max-latest--main": { @@ -2091,6 +2093,7 @@ "moe": True, "size": None, "act_param": None, + "reasoning": True, "date": "2025-01-28" }, "claude-3-7-sonnet-20250219--3200-output-128k-2025-02-19--main": { @@ -2121,6 +2124,10 @@ "open-data": "None", "prompted": True, "moe": False, + "size": 6.7, + "act_param": 6.7, + "open-data": "None", + "reasoning": False, "date": "2025-02-18" }, "google--gemma-3-27b-it--main": { @@ -2129,6 +2136,10 @@ "open-data": "None", "prompted": True, "moe": False, + "size": 27, + "act_param": 27, + "open-data": "None", + "reasoning": False, "date": "2025-03-12" }, "Qwen--QwQ-32B--skip_prefill--main": { @@ -2137,6 +2148,10 @@ "open-data": "None", "prompted": True, "moe": False, + "size": 32, + "act_param": 32, + "open-data": "None", + "reasoning": True, "date": "2025-03-06" }, "deepseek-chat-0324--main": { @@ -2144,7 +2159,23 @@ "link": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324", "open-data": "None", "prompted": True, - "moe": False, + "moe": True, + "size": 671, + "act_param": 37, + "open-data": "None", + "reasoning": True, "date": "2025-03-24" + }, + "gemini-2.5-pro-exp-03-25--main": { + "name": "Gemini-2.5-Pro-Exp-03-25", + "link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/", + "open-data": "None", + "prompted": True, + "moe": False, + "size": None, + "act_param": 37, + "open-data": "None", + "reasoning": True, + "date": "2025-03-25" } -} +} \ No newline at end of file From 9bd90fedee89d7dc3676838c75d9642cb0cd0702 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 1 Apr 2025 01:11:27 +0800 Subject: [PATCH 318/325] feat: use google genai --- Docker/Evaluate.Dockerfile | 2 +- bigcodebench/gen/util/google_request.py | 42 ++++++++++++++++--------- bigcodebench/provider/google.py | 9 +++--- setup.cfg | 2 +- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/Docker/Evaluate.Dockerfile b/Docker/Evaluate.Dockerfile index 90e7f40..8b2cdcd 100755 --- a/Docker/Evaluate.Dockerfile +++ b/Docker/Evaluate.Dockerfile @@ -54,7 +54,7 @@ RUN pip install \ rich \ accelerate \ anthropic \ - google-generativeai \ + google-genai \ mistralai \ openai \ e2b diff --git a/bigcodebench/gen/util/google_request.py b/bigcodebench/gen/util/google_request.py index 9e13607..5a76362 100644 --- a/bigcodebench/gen/util/google_request.py +++ b/bigcodebench/gen/util/google_request.py @@ -1,11 +1,12 @@ import time -import google.generativeai as genai +from google import genai from google.api_core.exceptions import GoogleAPICallError, ResourceExhausted def make_request( - client: genai.GenerativeModel, + model: str, + client: genai.Client, message: str, temperature: float, n: int, @@ -13,21 +14,34 @@ def make_request( ) -> genai.types.GenerateContentResponse: kwargs = {"temperature": temperature, "max_output_tokens": max_new_tokens} - if "-thinking-" in client.model_name: + if "-thinking-" in model: kwargs.pop("max_output_tokens") - - response = client.generate_content( - [{"role": "user", "parts": [message]}], - generation_config=genai.types.GenerationConfig( + + response = client.models.generate_content( + model=model, + contents=message, + config=genai.types.GenerateContentConfig( candidate_count=n, + safety_settings=[ + genai.types.SafetySetting( + category='HARM_CATEGORY_DANGEROUS_CONTENT', + threshold='BLOCK_NONE' + ), + genai.types.SafetySetting( + category='HARM_CATEGORY_SEXUALLY_EXPLICIT', + threshold='BLOCK_NONE' + ), + genai.types.SafetySetting( + category='HARM_CATEGORY_HATE_SPEECH', + threshold='BLOCK_NONE' + ), + genai.types.SafetySetting( + category='HARM_CATEGORY_HARASSMENT', + threshold='BLOCK_NONE' + ), + ], **kwargs - ), - safety_settings=[ - {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"}, - {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"}, - {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"}, - {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}, - ], + ), ) return response diff --git a/bigcodebench/provider/google.py b/bigcodebench/provider/google.py index 2194c47..e3b18ff 100644 --- a/bigcodebench/provider/google.py +++ b/bigcodebench/provider/google.py @@ -2,7 +2,7 @@ from typing import List from tqdm import tqdm -import google.generativeai as genai +from google import genai from bigcodebench.provider.base import DecoderBase from bigcodebench.gen.util.google_request import make_auto_request @@ -12,8 +12,8 @@ class GoogleDecoder(DecoderBase): def __init__(self, name: str, **kwargs): super().__init__(name, **kwargs) - genai.configure(api_key=os.getenv("GOOGLE_API_KEY")) - self.client = genai.GenerativeModel(name) + self.model = name + self.client = genai.Client(api_key=os.getenv("GOOGLE_API_KEY")) def codegen( self, prompts: List[str], do_sample: bool = True, num_samples: int = 200 @@ -34,7 +34,8 @@ def codegen( tokenizer=None, ) ret = make_auto_request( - self.client, + model=self.model, + client=self.client, message=message, n=num_samples, temperature=self.temperature, diff --git a/setup.cfg b/setup.cfg index cc20139..5907add 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = rich accelerate>=0.30.1 anthropic>=0.26.1 - google-generativeai>=0.5.4 + google-genai mistralai>=0.2.0,<1.0.0 openai>=1.11.1 e2b From 8fb8e2399822ebc2a998f00e2a28cbeeeff40c7f Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Mon, 7 Apr 2025 20:26:22 +0800 Subject: [PATCH 319/325] update model meta info and processing script --- analysis/get_results.py | 8 +- analysis/utils.py | 418 +++++++++++++++++++++------------------- 2 files changed, 227 insertions(+), 199 deletions(-) diff --git a/analysis/get_results.py b/analysis/get_results.py index 607615a..641c43b 100755 --- a/analysis/get_results.py +++ b/analysis/get_results.py @@ -4,7 +4,7 @@ import numpy as np from numpy import mean from glob import glob -from utils import * +from utils import model_info from tqdm import tqdm import pandas as pd import itertools @@ -48,6 +48,8 @@ def get_results(tids): "moe": info["moe"], "size": info["size"], "act_param": info["act_param"], + "date": info.get("date", None), + "prefill": info.get("prefill", False), # "direct_complete": info["direct_complete"], } @@ -249,7 +251,7 @@ def get_solve_rate(data_dict, task="complete"): def get_hf_ds(results): hf_dataset = {"model": [], "link": [], "moe": [], "size": [], "act_param": [], "type": [], #"lazy": [],# "direct_complete": [], - "complete": [], "instruct": []} + "complete": [], "instruct": [], "date": [], "prefill": []} for model, result in results.items(): hf_dataset["model"].append(model) @@ -261,6 +263,8 @@ def get_hf_ds(results): # hf_dataset["lazy"].append(result["lazy"]) hf_dataset["complete"].append(result["pass@1"]["complete"]) hf_dataset["instruct"].append(result["pass@1"]["instruct"]) + hf_dataset["date"].append(result["date"]) + hf_dataset["prefill"].append(result["prefill"]) # hf_dataset["direct_complete"].append(result["direct_complete"]) return Dataset.from_dict(hf_dataset) diff --git a/analysis/utils.py b/analysis/utils.py index 798499b..9aa7203 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -7,7 +7,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-12-04", }, "bigcode/starcoder2-15b-instruct-v0.1": { @@ -18,7 +18,7 @@ "size": 15, "act_param": 15, "open-data": "Full", - "reasoning": False, + "prefill": True, "date": "2024-04-30" }, "bigcode/starcoder2-3b": { @@ -29,7 +29,7 @@ "size": 3, "act_param": 3, "open-data": "Full", - "reasoning": False, + "prefill": True, "date": "2024-02-29" }, "bigcode/starcoder2-7b": { @@ -40,7 +40,7 @@ "size": 7, "act_param": 7, "open-data": "Full", - "reasoning": False, + "prefill": True, "date": "2024-02-29" }, "bigcode/starcoder2-15b": { @@ -51,7 +51,7 @@ "size": 15, "act_param": 15, "open-data": "Full", - "reasoning": False, + "prefill": True, "date": "2024-02-29" }, "Qwen/CodeQwen1.5-7B": { @@ -62,7 +62,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-16" }, "google/codegemma-2b": { @@ -73,7 +73,7 @@ "size": 2, "act_param": 2, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-10" }, "google/codegemma-7b": { @@ -84,7 +84,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-10" }, "google/codegemma-7b-it": { @@ -95,7 +95,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-10" }, "gpt-3.5-turbo-0125": { @@ -106,7 +106,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-01-25" }, "gpt-4o": { @@ -117,7 +117,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-13" }, "gpt-4-turbo-2024-04-09": { @@ -128,7 +128,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-09" }, "gpt-4-0613": { @@ -139,7 +139,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-13" }, "codellama/CodeLlama-7b-hf": { @@ -150,7 +150,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "codellama/CodeLlama-13b-hf": { @@ -161,7 +161,7 @@ "size": 13, "act_param": 13, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "codellama/CodeLlama-7b-Instruct-hf": { @@ -172,7 +172,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "codellama/CodeLlama-13b-Instruct-hf": { @@ -183,7 +183,7 @@ "size": 13, "act_param": 13, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "mistral-large-2402": { @@ -194,7 +194,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-02-26" }, "mistral-small-2402": { @@ -205,7 +205,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-02-26" }, "mistralai/Mixtral-8x22B-v0.1": { @@ -216,7 +216,7 @@ "size": 176, "act_param": 44, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-17" }, "mistralai/Mixtral-8x22B-Instruct-v0.1": { @@ -227,7 +227,7 @@ "size": 176, "act_param": 44, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-17" }, "codellama/CodeLlama-34b-hf": { @@ -238,7 +238,7 @@ "size": 34, "act_param": 34, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "codellama/CodeLlama-34b-Instruct-hf": { @@ -249,7 +249,7 @@ "size": 34, "act_param": 34, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "codellama/CodeLlama-70b-hf": { @@ -260,7 +260,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "codellama/CodeLlama-70b-Instruct-hf": { @@ -271,7 +271,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "Qwen/CodeQwen1.5-7B-Chat": { @@ -282,7 +282,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-16" }, "Qwen/Qwen1.5-110B-Chat": { @@ -293,7 +293,7 @@ "size": 110, "act_param": 110, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-26" }, "Qwen/Qwen1.5-72B-Chat": { @@ -304,7 +304,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-26" }, "Qwen/Qwen1.5-32B-Chat": { @@ -315,7 +315,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-26" }, "deepseek-ai/DeepSeek-V2-Chat": { @@ -326,7 +326,7 @@ "size": 236, "act_param": 21, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-06" }, "deepseek-ai/deepseek-coder-1.3b-base": { @@ -337,7 +337,7 @@ "size": 1.3, "act_param": 1.3, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-1.3b-instruct": { @@ -348,7 +348,7 @@ "size": 1.3, "act_param": 1.3, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-33b-base": { @@ -359,7 +359,7 @@ "size": 33, "act_param": 33, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-33b-instruct": { @@ -370,7 +370,7 @@ "size": 33, "act_param": 33, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-6.7b-base": { @@ -381,7 +381,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-10-28" }, "deepseek-ai/deepseek-coder-6.7b-instruct": { @@ -392,7 +392,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-10-28" }, "meta-llama/Meta-Llama-3-70B": { @@ -403,7 +403,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-18" }, "meta-llama/Meta-Llama-3-70B-Instruct": { @@ -414,7 +414,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-18" }, "meta-llama/Meta-Llama-3-8B": { @@ -425,7 +425,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-18" }, "meta-llama/Meta-Llama-3-8B-Instruct": { @@ -436,7 +436,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-18" }, "ibm-granite/granite-3b-code-instruct": { @@ -447,7 +447,7 @@ "size": 3, "act_param": 3, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "ibm-granite/granite-8b-code-instruct": { @@ -458,7 +458,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "ibm-granite/granite-20b-code-instruct": { @@ -469,7 +469,7 @@ "size": 20, "act_param": 20, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "ibm-granite/granite-34b-code-instruct": { @@ -480,7 +480,7 @@ "size": 34, "act_param": 34, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "ibm-granite/granite-3b-code-base": { @@ -491,7 +491,7 @@ "size": 3, "act_param": 3, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "ibm-granite/granite-8b-code-base": { @@ -502,7 +502,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "ibm-granite/granite-20b-code-base": { @@ -513,7 +513,7 @@ "size": 20, "act_param": 20, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "ibm-granite/granite-34b-code-base": { @@ -524,7 +524,7 @@ "size": 34, "act_param": 34, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-06" }, "claude-3-haiku-20240307": { @@ -535,7 +535,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-03-07" }, "claude-3-sonnet-20240229": { @@ -546,7 +546,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-02-29" }, "claude-3-opus-20240229": { @@ -557,7 +557,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-02-29" }, "01-ai/Yi-1.5-34B-Chat": { @@ -568,7 +568,7 @@ "size": 34, "act_param": 34, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-20" }, "01-ai/Yi-1.5-34B": { @@ -579,7 +579,7 @@ "size": 34, "act_param": 34, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-20" }, "01-ai/Yi-1.5-9B-Chat": { @@ -590,7 +590,7 @@ "size": 9, "act_param": 9, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-20" }, "01-ai/Yi-1.5-9B": { @@ -601,7 +601,7 @@ "size": 9, "act_param": 9, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-20" }, "01-ai/Yi-1.5-6B-Chat": { @@ -612,7 +612,7 @@ "size": 6, "act_param": 6, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-20" }, "01-ai/Yi-1.5-6B": { @@ -623,7 +623,7 @@ "size": 6, "act_param": 6, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-20" }, "Qwen/Qwen2-57B-A14B": { @@ -634,7 +634,7 @@ "size": 57, "act_param": 14, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-07" }, "Qwen/Qwen2-7B-Instruct": { @@ -645,7 +645,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-07" }, "Qwen/Qwen2-72B-Chat": { @@ -656,7 +656,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-07" }, "gemini-1.5-pro": { @@ -667,7 +667,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-14" }, "gemini-1.5-flash": { @@ -678,7 +678,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-14" }, "m-a-p/OpenCodeInterpreter-DS-33B": { @@ -689,7 +689,7 @@ "size": 33, "act_param": 33, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-02-22" }, "m-a-p/OpenCodeInterpreter-DS-6.7B": { @@ -700,7 +700,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-02-22" }, "m-a-p/OpenCodeInterpreter-DS-1.3B": { @@ -711,7 +711,7 @@ "size": 1.3, "act_param": 1.3, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-02-22" }, "microsoft/Phi-3-medium-128k-instruct": { @@ -722,7 +722,7 @@ "size": 14, "act_param": 14, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-21" }, "microsoft/Phi-3-small-128k-instruct": { @@ -733,7 +733,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-21" }, "codestral-2405": { @@ -744,7 +744,7 @@ "size": 22, "act_param": 22, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-23" }, "codestral-mamba-2407": { @@ -755,7 +755,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-16" }, "mistralai/Mistral-7B-Instruct-v0.3": { @@ -766,7 +766,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-22" }, "mistralai/Mistral-7B-v0.3": { @@ -777,7 +777,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-22" }, "CohereForAI/c4ai-command-r-plus": { @@ -788,7 +788,7 @@ "size": 104, "act_param": 104, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-04" }, "deepseek-coder": { @@ -799,7 +799,7 @@ "size": 236, "act_param": 21, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-06-17" }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": { @@ -810,7 +810,7 @@ "size": 16, "act_param": 2.4, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-17" }, "deepseek-ai/DeepSeek-Coder-V2-Lite-Base": { @@ -821,7 +821,7 @@ "size": 16, "act_param": 2.4, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-17" }, "claude-3-5-sonnet-20240620": { @@ -832,7 +832,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-06-20" }, "NousResearch/Hermes-2-Theta-Llama-3-70B": { @@ -843,7 +843,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-24" }, "microsoft/wavecoder-ultra-6.7b": { @@ -854,7 +854,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-12-26" }, "google/gemma-2-9b-it": { @@ -865,7 +865,7 @@ "size": 9, "act_param": 9, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-19" }, "Bin12345/AutoCoder": { @@ -876,7 +876,7 @@ "size": 33, "act_param": 33, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-23" }, "Bin12345/AutoCoder_S_6.7B": { @@ -887,7 +887,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-23" }, "Bin12345/AutoCoder_QW_7B": { @@ -898,7 +898,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-23" }, "SenseLLM/ReflectionCoder-DS-33B": { @@ -909,7 +909,7 @@ "size": 33, "act_param": 33, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-05-27" }, "SenseLLM/ReflectionCoder-DS-6.7B": { @@ -920,7 +920,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-05-27" }, "SenseLLM/ReflectionCoder-CL-34B": { @@ -931,7 +931,7 @@ "size": 34, "act_param": 34, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-05-27" }, "SenseLLM/ReflectionCoder-CL-7B": { @@ -942,7 +942,7 @@ "size": 7, "act_param": 7, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-05-27" }, "new-microsoft/Phi-3-mini-128k-instruct": { @@ -953,7 +953,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-21" }, "old-microsoft/Phi-3-mini-128k-instruct": { @@ -964,7 +964,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-21" }, "internlm/internlm2_5-7b-chat": { @@ -975,7 +975,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-03" }, "NousResearch/Hermes-2-Pro-Llama-3-70B": { @@ -986,7 +986,7 @@ "size": 70, "act_param": 70, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-06-27" }, "new-deepseek-chat": { @@ -997,7 +997,7 @@ "size": 236, "act_param": 21, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-06-28" }, "vllm-google/gemma-2-27b-it": { @@ -1008,7 +1008,7 @@ "size": 27, "act_param": 27, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-06-19" }, "Artigenz/Artigenz-Coder-DS-6.7B": { @@ -1019,7 +1019,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-16" }, "openchat/openchat-3.6-8b-20240522": { @@ -1030,7 +1030,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-22" }, "Phind/Phind-CodeLlama-34B-v2": { @@ -1041,7 +1041,7 @@ "size": 34, "act_param": 34, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2023-08-25" }, "yi-large": { @@ -1052,7 +1052,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-13" }, "THUDM/codegeex4-all-9b": { @@ -1063,7 +1063,7 @@ "size": 9, "act_param": 9, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-05" }, "gpt-4o-mini-2024-07-18": { @@ -1074,7 +1074,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-18" }, "Nexusflow/Athene-70B": { @@ -1085,7 +1085,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-20" }, "NTQAI/Nxcode-CQ-7B-orpo": { @@ -1096,7 +1096,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-25" }, "migtissera/Llama-3-70B-Synthia-v3.5": { @@ -1107,7 +1107,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-05-27" }, "migtissera/Tess-v2.5.2-Qwen2-72B": { @@ -1118,7 +1118,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-18" }, "WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5": { @@ -1129,7 +1129,7 @@ "size": 33, "act_param": 33, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-02-10" }, "mistral-large-2407": { @@ -1140,7 +1140,7 @@ "size": 123, "act_param": 123, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-07-24" }, "meta-llama/Meta-Llama-3.1-8B-Instruct": { @@ -1151,7 +1151,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-23" }, "meta-llama/Meta-Llama-3.1-70B-Instruct": { @@ -1162,7 +1162,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-23" }, "meta--llama-3.1-405b-instruct": { @@ -1173,7 +1173,7 @@ "size": 405, "act_param": 405, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-23" }, "deepseek-coder-20240724": { @@ -1184,7 +1184,7 @@ "size": 236, "act_param": 21, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-24" }, "microsoft/Phi-3.5-mini-instruct": { @@ -1195,7 +1195,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-23" }, "nv-mistralai--mistral-nemo-12b-instruct": { @@ -1206,7 +1206,7 @@ "size": 12, "act_param": 12, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-07-18" }, "wyt2000/InverseCoder-CL-13B": { @@ -1217,7 +1217,7 @@ "size": 13, "act_param": 13, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-07-08" }, "wyt2000/InverseCoder-CL-7B": { @@ -1228,7 +1228,7 @@ "size": 7, "act_param": 7, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-07-08" }, "wyt2000/InverseCoder-DS-6.7B": { @@ -1239,7 +1239,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-07-08" }, "gemini-1.5-pro-exp-0801": { @@ -1250,7 +1250,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-08-01" }, "gpt-4o-2024-08-06": { @@ -1261,7 +1261,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-06" }, "abacusai/Dracarys-Llama-3.1-70B-Instruct": { @@ -1272,7 +1272,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-23" }, "abacusai/Dracarys-72B-Instruct": { @@ -1283,7 +1283,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-23" }, "gemini-1.5-pro-exp-0827": { @@ -1294,7 +1294,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-27" }, "gemini-1.5-flash-exp-0827": { @@ -1305,7 +1305,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-27" }, "microsoft/Phi-3.5-mini-instruct": { @@ -1316,7 +1316,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-23" }, "abacusai/Dracarys-Llama-3.1-70B-Instruct": { @@ -1327,7 +1327,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-23" }, "abacusai/Dracarys-72B-Instruct": { @@ -1338,7 +1338,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-04-23" }, "deepseek-coder-v2.5": { @@ -1349,7 +1349,7 @@ "size": 236, "act_param": 21, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-18" }, "CohereForAI/c4ai-command-r-08-2024": { @@ -1360,7 +1360,7 @@ "size": 32.3, "act_param": 32.3, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-30" }, "CohereForAI/c4ai-command-r-plus-08-2024": { @@ -1371,7 +1371,7 @@ "size": 104, "act_param": 104, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-30" }, "ayueei--yue-coder-9b-preview": { @@ -1382,7 +1382,7 @@ "size": 9, "act_param": 9, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-04" }, # "mattshumer/ref_70_e3_prefill": { @@ -1411,7 +1411,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-09-12" }, "o1-mini-2024-09-12": { @@ -1422,7 +1422,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-09-12" }, "Qwen/Qwen2.5-Coder-1.5B-Instruct": { @@ -1433,7 +1433,7 @@ "size": 1.5, "act_param": 1.5, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-12" }, "Qwen/Qwen2.5-Coder-7B-Instruct": { @@ -1444,7 +1444,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-12" }, "gemini-1.5-pro-002": { @@ -1455,7 +1455,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-09-25" }, "mistralai/Mistral-Small-Instruct-2409": { @@ -1466,7 +1466,7 @@ "size": 22.2, "act_param": 22.2, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-18" }, "Qwen/Qwen2.5-0.5B-Instruct": { @@ -1477,7 +1477,7 @@ "size": 0.5, "act_param": 0.5, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "Qwen/Qwen2.5-1.5B-Instruct": { @@ -1488,7 +1488,7 @@ "size": 1.5, "act_param": 1.5, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "Qwen/Qwen2.5-7B-Instruct": { @@ -1499,7 +1499,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "Qwen/Qwen2.5-14B-Instruct": { @@ -1510,7 +1510,7 @@ "size": 14, "act_param": 14, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "Qwen/Qwen2.5-32B-Instruct": { @@ -1521,7 +1521,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "Qwen/Qwen2.5-72B-Instruct": { @@ -1532,7 +1532,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "meta-llama/Llama-3.2-1B-Instruct": { @@ -1543,7 +1543,7 @@ "size": 1, "act_param": 1, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-25" }, "meta-llama/Llama-3.2-3B-Instruct": { @@ -1554,7 +1554,7 @@ "size": 3, "act_param": 3, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-25" }, "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { @@ -1565,7 +1565,7 @@ "size": 70, "act_param": 70, "open-data": "Partial", - "reasoning": False, + "prefill": True, "date": "2024-09-25" }, "claude-3-5-sonnet-20241022": { @@ -1576,7 +1576,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-10-22" }, "ibm-granite/granite-3.0-8b-instruct": { @@ -1587,7 +1587,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-10-21" }, "ibm-granite/granite-3.0-2b-instruct": { @@ -1598,7 +1598,7 @@ "size": 2, "act_param": 2, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-10-21" }, "grok-beta--main": { @@ -1609,7 +1609,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-03-17" }, "claude-3-5-haiku-20241022--main": { @@ -1620,7 +1620,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-10-22" }, "Qwen/Qwen2.5-Coder-14B-Instruct--main": { @@ -1631,7 +1631,7 @@ "size": 14, "act_param": 14, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "Qwen/Qwen2.5-Coder-32B-Instruct--main": { @@ -1642,7 +1642,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-09-19" }, "infly/OpenCoder-1.5B-Instruct--main": { @@ -1653,7 +1653,7 @@ "size": 1.5, "act_param": 1.5, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-09" }, "infly/OpenCoder-8B-Instruct--main": { @@ -1664,7 +1664,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-09" }, "microsoft/Phi-3.5-mini-instruct--main": { @@ -1675,7 +1675,7 @@ "size": 3.8, "act_param": 3.8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-08-21" }, "Nexusflow/Athene-V2-Agent--main": { @@ -1686,7 +1686,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-14" }, "Nexusflow/Athene-V2-Chat--main": { @@ -1697,7 +1697,7 @@ "size": 72, "act_param": 72, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-14" }, "gemini-exp-1114--main": { @@ -1708,7 +1708,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-11-14" }, "gpt-4o-2024-11-20--main": { @@ -1719,7 +1719,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-11-20" }, "gemini-exp-1121--main": { @@ -1730,7 +1730,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-21" }, "gemini-exp-1206--main": { @@ -1741,7 +1741,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-12-06" }, "meta-llama--Llama-3.3-70B-Instruct--main": { @@ -1752,7 +1752,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-12-19" }, "deepseek-ai--DeepSeek-V2.5-1210--main": { @@ -1763,7 +1763,7 @@ "size": 236, "act_param": 21, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-12-10" }, "gemini-2.0-flash-exp--main": { @@ -1774,7 +1774,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-12-11" }, "gemini-2.0-flash-thinking-exp-1219--main": { @@ -1785,7 +1785,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-12-19" }, "gemini-2.0-flash-thinking-exp-01-21--main": { @@ -1796,7 +1796,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-21" }, "o1-2024-12-17--main": { @@ -1807,7 +1807,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-12-17" }, "o1-2024-12-17--low--main": { @@ -1818,7 +1818,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-12-17" }, "o1-2024-12-17--high--main": { @@ -1829,7 +1829,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-12-17" }, "deepseek-v3-chat--main": { @@ -1840,7 +1840,7 @@ "size": 671, "act_param": 37, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2024-12-26" }, "microsoft--phi-4--main": { @@ -1851,7 +1851,7 @@ "size": 14.7, "act_param": 14.7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-12-13" }, "deepseek-reasoner--main": { @@ -1862,7 +1862,7 @@ "size": 671, "act_param": 37, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Llama-70B--main": { @@ -1873,7 +1873,7 @@ "size": 70, "act_param": 70, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B--main": { @@ -1884,7 +1884,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B--main": { @@ -1895,7 +1895,7 @@ "size": 14, "act_param": 14, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Llama-8B--main": { @@ -1906,7 +1906,7 @@ "size": 8, "act_param": 8, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B--main": { @@ -1917,7 +1917,7 @@ "size": 7, "act_param": 7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-20" }, "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B--main": { @@ -1928,7 +1928,7 @@ "size": 1.5, "act_param": 1.5, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-20" }, "mistralai/Mistral-Small-24B-Instruct-2501--main": { @@ -1939,7 +1939,7 @@ "size": 24, "act_param": 24, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-31" }, "o3-mini-2025-01-31--medium--main": { @@ -1950,7 +1950,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-01-31" }, "o3-mini-2025-01-31--low--main": { @@ -1961,7 +1961,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-01-31" }, "o3-mini-2025-01-31--high--main": { @@ -1972,7 +1972,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-01-31" }, "gemini-2.0-flash-001--main": { @@ -1983,7 +1983,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-02-05" }, "gemini-2.0-flash-exp--main": { @@ -1994,7 +1994,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-02-05" }, "gemini-2.0-flash-lite-preview-02-05--main": { @@ -2005,7 +2005,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-02-05" }, "gemini-2.0-pro-exp-02-05--main": { @@ -2016,7 +2016,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-02-05" }, "NovaSky-AI--Sky-T1-32B-Flash--main": { @@ -2027,7 +2027,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-12" }, "NovaSky-AI--Sky-T1-32B-Preview--main": { @@ -2038,7 +2038,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-01-12" }, "Qwen--QwQ-32B-Preview--main": { @@ -2049,7 +2049,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2024-11-28" }, "claude-3-7-sonnet-20250219--main": { @@ -2060,7 +2060,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-02-19" }, "chatgpt-4o-latest--main": { @@ -2071,7 +2071,7 @@ "moe": False, "size": None, "act_param": None, - "reasoning": True, + "prefill": False, "date": "2025-01-29" }, "Kwaipilot--KwaiCoder-23B-A4B-v1--main": { @@ -2082,7 +2082,7 @@ "moe": True, "size": 23, "act_param": 4, - "reasoning": False, + "prefill": True, "date": "2025-01-25" }, "qwen-max-latest--main": { @@ -2093,7 +2093,7 @@ "moe": True, "size": None, "act_param": None, - "reasoning": True, + "prefill": False, "date": "2025-01-28" }, "claude-3-7-sonnet-20250219--3200-output-128k-2025-02-19--main": { @@ -2104,7 +2104,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-02-19" }, "claude-3-7-sonnet-20250219--main": { @@ -2115,7 +2115,7 @@ "size": None, "act_param": None, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-02-19" }, "WarriorCoder-6.7B--main": { @@ -2127,7 +2127,7 @@ "size": 6.7, "act_param": 6.7, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-02-18" }, "google--gemma-3-27b-it--main": { @@ -2139,7 +2139,7 @@ "size": 27, "act_param": 27, "open-data": "None", - "reasoning": False, + "prefill": True, "date": "2025-03-12" }, "Qwen--QwQ-32B--skip_prefill--main": { @@ -2151,7 +2151,7 @@ "size": 32, "act_param": 32, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-03-06" }, "deepseek-chat-0324--main": { @@ -2163,7 +2163,7 @@ "size": 671, "act_param": 37, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-03-24" }, "gemini-2.5-pro-exp-03-25--main": { @@ -2175,7 +2175,31 @@ "size": None, "act_param": 37, "open-data": "None", - "reasoning": True, + "prefill": False, "date": "2025-03-25" - } + }, + "meta/llama-4-scout-17b-16e-instruct--main": { + "name": "Llama-4-Scout", + "link": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct", + "open-data": "None", + "prompted": True, + "moe": True, + "size": 109, + "act_param": 17, + "open-data": "None", + "prefill": False, + "date": "2025-04-05" + }, + "meta/llama-4-maverick-17b-128e-instruct--main": { + "name": "Llama-4-Maverick", + "link": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct", + "open-data": "None", + "prompted": True, + "moe": True, + "size": 109, + "act_param": 17, + "open-data": "None", + "prefill": False, + "date": "2025-04-05" + }, } \ No newline at end of file From 1bf199d4a88d73b0940c973a4b6d2c1d86503179 Mon Sep 17 00:00:00 2001 From: Alessandro Giagnorio Date: Thu, 10 Apr 2025 21:28:53 +0200 Subject: [PATCH 320/325] Fix nltk resource download --- tools/fix_v025.py | 135 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 tools/fix_v025.py diff --git a/tools/fix_v025.py b/tools/fix_v025.py new file mode 100644 index 0000000..902fad1 --- /dev/null +++ b/tools/fix_v025.py @@ -0,0 +1,135 @@ +from datasets import load_dataset +from huggingface_hub import HfApi + +BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_HARD_HF = "bigcode/bigcodebench-hard" +BIGCODEBENCH_VERSION = "v0.1.4" +BIGCODEBENCH_UPDATE = "bigcode/bcb_update" +BIGCODEBENCH_NEW_VERSION = "v0.1.5" + +def map_ds(sample): + if sample["task_id"] in ["BigCodeBench/332"]: + sample['code_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['code_prompt'] + sample['complete_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['complete_prompt'] + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('stopwords')\n" + ) + + if sample["task_id"] in ["BigCodeBench/334"]: + sample['code_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['code_prompt'] + sample['complete_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['complete_prompt'] + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('punkt')\n" + ) + + if sample["task_id"] in ["BigCodeBench/376"]: + sample['code_prompt'] = sample['code_prompt'].replace( + "import nltk\n", + "import nltk\nnltk.download('stopwords')\n", + 1 + ) + sample['complete_prompt'] = sample['complete_prompt'].replace( + "import nltk\n", + "import nltk\nnltk.download('stopwords')\n", + 1 + ) + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\nimport nltk\n", + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('stopwords')\n" + ) + + if sample["task_id"] in ["BigCodeBench/383"]: + sample['code_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['code_prompt'] + sample['complete_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['complete_prompt'] + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('punkt')\n" + ) + + if sample["task_id"] in ["BigCodeBench/633"]: + sample['code_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['code_prompt'] + sample['complete_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['complete_prompt'] + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\n" + ) + + if sample["task_id"] in ["BigCodeBench/635"]: + sample['code_prompt'] = sample['code_prompt'].replace( + "# Importing the required libraries", + "# Importing the required libraries\nimport nltk\nnltk.download('stopwords')\n" + ) + + sample['complete_prompt'] = sample['complete_prompt'].replace( + "# Importing the required libraries", + "# Importing the required libraries\nimport nltk\nnltk.download('stopwords')\n" + ) + + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "# Importing the required libraries", + "# Importing the required libraries\nimport nltk\nnltk.download('stopwords')\n" + ) + + if sample["task_id"] in ["BigCodeBench/849"]: + sample['code_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['code_prompt'] + sample['complete_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['complete_prompt'] + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\n" + ) + + if sample["task_id"] in ["BigCodeBench/940"]: + sample['code_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['code_prompt'] + sample['complete_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['complete_prompt'] + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('punkt')\n" + ) + + if sample["task_id"] in ["BigCodeBench/1109"]: + sample['code_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['code_prompt'] + sample['complete_prompt'] = "import nltk\nnltk.download('punkt')\n" + sample['complete_prompt'] + sample['instruct_prompt'] = sample['instruct_prompt'].replace( + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('punkt')\n" + ) + + return sample + +if __name__ == "__main__": + api = HfApi() + ds_dict = load_dataset(BIGCODEBENCH_HF) + hard_ds_dict = load_dataset(BIGCODEBENCH_HARD_HF) + ds = ds_dict[BIGCODEBENCH_VERSION] + hard_ds = hard_ds_dict[BIGCODEBENCH_VERSION] + function_id = [332, 334, 376, 383, 633, 635, 849, 940, 1109] + + new_ds = ds.map(map_ds) + new_ds.to_json("BigCodeBench.jsonl") + ds_dict[BIGCODEBENCH_NEW_VERSION] = new_ds + ds_dict.push_to_hub(BIGCODEBENCH_HF) + + new_hard_ds = hard_ds.map(map_ds) + new_hard_ds.to_json("BigCodeBench-Hard.jsonl") + hard_ds_dict[BIGCODEBENCH_NEW_VERSION] = new_hard_ds + hard_ds_dict.push_to_hub(BIGCODEBENCH_HARD_HF) + + for i in function_id: + old_sample = ds.select([i]) + new_sample = new_ds.select([i]) + old_sample.to_json("old.jsonl") + new_sample.to_json("new.jsonl") + api.upload_file( + path_or_fileobj="old.jsonl", + path_in_repo=f"{i}/old.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) + api.upload_file( + path_or_fileobj="new.jsonl", + path_in_repo=f"{i}/new.jsonl", + repo_id=BIGCODEBENCH_UPDATE, + # repo_type="dataset" + ) \ No newline at end of file From 821f3a54e5b3fd285cfe8b32956dbff56b432a86 Mon Sep 17 00:00:00 2001 From: Alessandro Giagnorio Date: Thu, 10 Apr 2025 21:49:30 +0200 Subject: [PATCH 321/325] Update instruction prompts with ntlk fix --- tools/fix_v025.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/fix_v025.py b/tools/fix_v025.py index 902fad1..edbeb71 100644 --- a/tools/fix_v025.py +++ b/tools/fix_v025.py @@ -53,7 +53,7 @@ def map_ds(sample): sample['complete_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['complete_prompt'] sample['instruct_prompt'] = sample['instruct_prompt'].replace( "\nYou should write self-contained code starting with:\n```\n", - "\nYou should write self-contained code starting with:\n```\n" + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('stopwords')\n" ) if sample["task_id"] in ["BigCodeBench/635"]: @@ -68,8 +68,8 @@ def map_ds(sample): ) sample['instruct_prompt'] = sample['instruct_prompt'].replace( - "# Importing the required libraries", - "# Importing the required libraries\nimport nltk\nnltk.download('stopwords')\n" + "\nYou should write self-contained code starting with:\n```\n", + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('stopwords')\n" ) if sample["task_id"] in ["BigCodeBench/849"]: @@ -77,7 +77,7 @@ def map_ds(sample): sample['complete_prompt'] = "import nltk\nnltk.download('stopwords')\n" + sample['complete_prompt'] sample['instruct_prompt'] = sample['instruct_prompt'].replace( "\nYou should write self-contained code starting with:\n```\n", - "\nYou should write self-contained code starting with:\n```\n" + "\nYou should write self-contained code starting with:\n```\nimport nltk\nnltk.download('stopwords')\n" ) if sample["task_id"] in ["BigCodeBench/940"]: From bb082968da15445403a1a33b3cb238b5c9531b47 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 11 Apr 2025 18:04:45 +0800 Subject: [PATCH 322/325] feat: support reasoning for grok-3-mini --- bigcodebench/gen/util/openai_request.py | 2 +- bigcodebench/generate.py | 2 +- bigcodebench/provider/openai.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index f8db3f5..f14f6d1 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -17,7 +17,7 @@ def make_request( kwargs["top_p"] = 0.95 kwargs["max_completion_tokens"] = max_tokens kwargs["temperature"] = temperature - if model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): # pop top-p and max_completion_tokens + if any(model.startswith(m) or model.endswith(m) for m in ["o1-", "o3-", "reasoner", "grok-3-mini"]): # pop top-p and max_completion_tokens kwargs.pop("top_p") kwargs.pop("max_completion_tokens") kwargs.pop("temperature") diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 912abcd..7eeecfc 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -197,7 +197,7 @@ def run_codegen( ) extra = "-" + subset if subset != "full" else "" - if backend == "openai" and reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"): + if backend == "openai" and reasoning_effort and any(model.startswith(m) or model.endswith(m) for m in ["o1-", "o3-", "reasoner", "grok-3-mini"]): model = model + f"--{reasoning_effort}" if lora_path: diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 12790f6..8b187d1 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -28,7 +28,7 @@ def codegen( tokenizer=None, ) for prompt in prompts] # use concurrency based batching for o1 and deepseek models - if self.name.startswith("o1-") or self.name.startswith("o3-") or self.name.startswith("deepseek"): + if any(self.name.startswith(model) or self.name.endswith(model) for model in ["o1-", "o3-", "reasoner", "grok-3-mini"]): return self._codegen_batch_via_concurrency(messages, num_samples) return self._codegen_api_batch(messages, num_samples) @@ -49,6 +49,7 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] reasoning_effort=self.reasoning_effort, n=num_samples, ) + print(ret) outputs = [] for item in ret.choices: outputs.append(item.message.content) From 33ed54d44343b1e61c1f2a777833130a0b57b2b7 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 11 Apr 2025 23:03:57 +0800 Subject: [PATCH 323/325] fix: update grok3 name for reasoning --- bigcodebench/gen/util/openai_request.py | 2 +- bigcodebench/generate.py | 2 +- bigcodebench/provider/openai.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bigcodebench/gen/util/openai_request.py b/bigcodebench/gen/util/openai_request.py index f14f6d1..3c8b741 100644 --- a/bigcodebench/gen/util/openai_request.py +++ b/bigcodebench/gen/util/openai_request.py @@ -17,7 +17,7 @@ def make_request( kwargs["top_p"] = 0.95 kwargs["max_completion_tokens"] = max_tokens kwargs["temperature"] = temperature - if any(model.startswith(m) or model.endswith(m) for m in ["o1-", "o3-", "reasoner", "grok-3-mini"]): # pop top-p and max_completion_tokens + if any(model.startswith(m) or model.endswith(m) for m in ["o1-", "o3-", "reasoner", "grok-3-mini-beta"]): # pop top-p and max_completion_tokens kwargs.pop("top_p") kwargs.pop("max_completion_tokens") kwargs.pop("temperature") diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index 7eeecfc..adbf892 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -197,7 +197,7 @@ def run_codegen( ) extra = "-" + subset if subset != "full" else "" - if backend == "openai" and reasoning_effort and any(model.startswith(m) or model.endswith(m) for m in ["o1-", "o3-", "reasoner", "grok-3-mini"]): + if backend == "openai" and reasoning_effort and any(model.startswith(m) or model.endswith(m) for m in ["o1-", "o3-", "reasoner", "grok-3-mini-beta"]): model = model + f"--{reasoning_effort}" if lora_path: diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 8b187d1..046e13e 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -28,7 +28,7 @@ def codegen( tokenizer=None, ) for prompt in prompts] # use concurrency based batching for o1 and deepseek models - if any(self.name.startswith(model) or self.name.endswith(model) for model in ["o1-", "o3-", "reasoner", "grok-3-mini"]): + if any(self.name.startswith(model) or self.name.endswith(model) for model in ["o1-", "o3-", "reasoner", "grok-3-mini-beta"]): return self._codegen_batch_via_concurrency(messages, num_samples) return self._codegen_api_batch(messages, num_samples) From 10c8327f3ef5d94644e84233b7400e3fb3178e4d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 11 Apr 2025 23:05:02 +0800 Subject: [PATCH 324/325] add models --- analysis/utils.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/analysis/utils.py b/analysis/utils.py index 9aa7203..20ecbf5 100755 --- a/analysis/utils.py +++ b/analysis/utils.py @@ -2202,4 +2202,52 @@ "prefill": False, "date": "2025-04-05" }, + "agentica-org/DeepCoder-14B-Preview--main": { + "name": "DeepCoder-14B-Preview", + "link": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview", + "open-data": "None", + "prompted": True, + "moe": True, + "size": 14, + "act_param": 14, + "open-data": "None", + "prefill": True, + "date": "2025-04-09" + }, + "openrouter/quasar-alpha--main": { + "name": "Quasar-Alpha", + "link": "https://openrouter.ai/openrouter/quasar-alpha", + "open-data": "None", + "prompted": True, + "moe": True, + "size": None, + "act_param": None, + "open-data": "None", + "prefill": False, + "date": "2025-04-02" + }, + "agentica-org/DeepCoder-14B-Preview--skip_prefill--main": { + "name": "DeepCoder-14B-Preview (w/ Reasoning, 64k tokens, temperature=0.6)", + "link": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview", + "open-data": "None", + "prompted": True, + "moe": False, + "size": 14, + "act_param": 14, + "open-data": "None", + "prefill": False, + "date": "2025-04-09" + }, + "openrouter/optimus-alpha--main": { + "name": "Optimus-Alpha", + "link": "https://openrouter.ai/openrouter/optimus-alpha", + "open-data": "None", + "prompted": True, + "moe": True, + "size": None, + "act_param": None, + "open-data": "None", + "prefill": False, + "date": "2025-04-10" + } } \ No newline at end of file From 77b286f79e43a898b52ab96f48e25fa96fab843d Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Fri, 11 Apr 2025 23:09:06 +0800 Subject: [PATCH 325/325] fix: rm printout --- bigcodebench/provider/openai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 046e13e..ff1459f 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -49,7 +49,6 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] reasoning_effort=self.reasoning_effort, n=num_samples, ) - print(ret) outputs = [] for item in ret.choices: outputs.append(item.message.content)