Skip to content

Commit 25e2e6c

Browse files
JeronymousJeronymous
committed
Benchmark many LLMs
1 parent b50265a commit 25e2e6c

File tree

1 file changed

+86
-36
lines changed

1 file changed

+86
-36
lines changed

‎expes/tokenizers_fertilities.json‎

Lines changed: 86 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,122 @@
11
{
22
"wikimedia/wikipedia/20231101.de": {
3-
"gpt-4": 2.7174367093937914,
4-
"google/gemma-7b": 2.438275867877175,
5-
"microsoft/Phi-3-medium-4k-instruct": 2.993447758931429,
6-
"mistralai/Mistral-7B-v0.1": 3.1795634085128603,
7-
"meta-llama/Meta-Llama-3-8B-Instruct": 2.7094359008910374,
3+
"CohereForAI/aya-expanse-8b": 2.4974144755680574,
4+
"CohereForAI/c4ai-command-r-plus": 2.4974144755680574,
5+
"HuggingFaceH4/zephyr-7b-beta": 3.1795634085128603,
6+
"OpenLLM-France/Lucie-7B": 2.7991207532550657,
87
"Qwen/Qwen2.5-7B": 2.90382185989321,
8+
"allenai/OLMo-2-1124-7B-Instruct": 2.7920463541578937,
99
"bigscience/bloom-7b1": 2.6884569388064476,
1010
"croissantllm/CroissantLLMBase": 3.4861880779531407,
11-
"utter-project/EuroLLM-9B": 2.580058616449662,
12-
"OpenLLM-France/Lucie-7B": 2.7991207532550657,
13-
"CohereForAI/c4ai-command-r-plus": 2.4974144755680574,
14-
"allenai/OLMo-2-1124-7B-Instruct": 2.7920463541578937
11+
"google/gemma-7b": 2.438275867877175,
12+
"gpt-3.5-turbo": 2.7174367093937914,
13+
"gpt-4": 2.7174367093937914,
14+
"inceptionai/jais-adapted-7b-chat": 2.993447758931429,
15+
"meta-llama/Llama-2-7b-hf": 2.993447758931429,
16+
"meta-llama/Meta-Llama-3-8B-Instruct": 2.7094359008910374,
17+
"microsoft/Phi-3-medium-4k-instruct": 2.993447758931429,
18+
"microsoft/phi-2": 3.259655712577271,
19+
"mistralai/Mistral-7B-Instruct-v0.3": 3.1795634085128603,
20+
"mistralai/Mistral-7B-v0.1": 3.1795634085128603,
21+
"nvidia/parakeet-ctc-1.1b": 4.315282386430629,
22+
"openai/whisper-large-v3": 2.740201115060048,
23+
"tiiuae/falcon-7b": 2.8115009516751166,
24+
"utter-project/EuroLLM-9B": 2.580058616449662
1525
},
1626
"wikimedia/wikipedia/20231101.en": {
27+
"CohereForAI/aya-expanse-8b": 1.3982310275520267,
1728
"CohereForAI/c4ai-command-r-plus": 1.3982310275520267,
29+
"HuggingFaceH4/zephyr-7b-beta": 1.6146211752368824,
1830
"OpenLLM-France/Lucie-7B": 1.5008114552513638,
1931
"Qwen/Qwen2.5-7B": 1.4221377476499009,
2032
"allenai/OLMo-2-1124-7B-Instruct": 1.4215510030835299,
2133
"bigscience/bloom-7b1": 1.3755539742581426,
2234
"croissantllm/CroissantLLMBase": 1.5767012471443018,
2335
"google/gemma-7b": 1.3943485262724242,
36+
"gpt-3.5-turbo": 1.3809220628440881,
2437
"gpt-4": 1.3809220628440881,
38+
"inceptionai/jais-adapted-7b-chat": 1.647653645930864,
39+
"meta-llama/Llama-2-7b-hf": 1.64767237182128,
2540
"meta-llama/Meta-Llama-3-8B-Instruct": 1.3791118934371995,
2641
"microsoft/Phi-3-medium-4k-instruct": 1.64767237182128,
42+
"microsoft/phi-2": 1.3964583099259704,
43+
"mistralai/Mistral-7B-Instruct-v0.3": 1.6146211752368824,
2744
"mistralai/Mistral-7B-v0.1": 1.6146211752368824,
45+
"nvidia/parakeet-ctc-1.1b": 2.3931438273223224,
46+
"openai/whisper-large-v3": 1.4640400484376366,
47+
"tiiuae/falcon-7b": 1.4495524512190554,
2848
"utter-project/EuroLLM-9B": 1.5209480294121318
2949
},
3050
"wikimedia/wikipedia/20231101.es": {
31-
"gpt-4": 1.8225358585955305,
32-
"google/gemma-7b": 1.5613474022865241,
33-
"microsoft/Phi-3-medium-4k-instruct": 1.9952571541648314,
34-
"mistralai/Mistral-7B-v0.1": 2.0873181536733116,
35-
"meta-llama/Meta-Llama-3-8B-Instruct": 1.815680116432716,
51+
"CohereForAI/aya-expanse-8b": 1.5383225773505174,
52+
"CohereForAI/c4ai-command-r-plus": 1.5383225773505174,
53+
"HuggingFaceH4/zephyr-7b-beta": 2.0873181536733116,
54+
"OpenLLM-France/Lucie-7B": 1.7532315823742317,
3655
"Qwen/Qwen2.5-7B": 1.8497290258335088,
56+
"allenai/OLMo-2-1124-7B-Instruct": 1.8630702745488552,
3757
"bigscience/bloom-7b1": 1.458626170550949,
3858
"croissantllm/CroissantLLMBase": 2.26073523685504,
39-
"utter-project/EuroLLM-9B": 1.6609343980798816,
40-
"OpenLLM-France/Lucie-7B": 1.7532315823742317,
41-
"CohereForAI/c4ai-command-r-plus": 1.5383225773505174,
42-
"allenai/OLMo-2-1124-7B-Instruct": 1.8630702745488552
59+
"google/gemma-7b": 1.5613474022865241,
60+
"gpt-3.5-turbo": 1.8225358585955305,
61+
"gpt-4": 1.8225358585955305,
62+
"inceptionai/jais-adapted-7b-chat": 1.9952252373018762,
63+
"meta-llama/Llama-2-7b-hf": 1.9952571541648314,
64+
"meta-llama/Meta-Llama-3-8B-Instruct": 1.815680116432716,
65+
"microsoft/Phi-3-medium-4k-instruct": 1.9952571541648314,
66+
"microsoft/phi-2": 2.275065908322003,
67+
"mistralai/Mistral-7B-Instruct-v0.3": 2.0873181536733116,
68+
"mistralai/Mistral-7B-v0.1": 2.0873181536733116,
69+
"nvidia/parakeet-ctc-1.1b": 3.199537843824406,
70+
"openai/whisper-large-v3": 1.8791819069687279,
71+
"tiiuae/falcon-7b": 1.904287711369425,
72+
"utter-project/EuroLLM-9B": 1.6609343980798816
4373
},
4474
"wikimedia/wikipedia/20231101.fr": {
45-
"gpt-4": 1.8529395175694223,
46-
"google/gemma-7b": 1.6441557464889356,
47-
"microsoft/Phi-3-medium-4k-instruct": 1.996634642501269,
48-
"mistralai/Mistral-7B-v0.1": 2.0805805711707306,
49-
"meta-llama/Meta-Llama-3-8B-Instruct": 1.8492545451127114,
75+
"CohereForAI/aya-expanse-8b": 1.586136231175619,
76+
"CohereForAI/c4ai-command-r-plus": 1.586136231175619,
77+
"HuggingFaceH4/zephyr-7b-beta": 2.0805805711707306,
78+
"OpenLLM-France/Lucie-7B": 1.5905920397074582,
5079
"Qwen/Qwen2.5-7B": 1.8761398033427965,
80+
"allenai/OLMo-2-1124-7B-Instruct": 1.9298413832434025,
5181
"bigscience/bloom-7b1": 1.422223893411545,
5282
"croissantllm/CroissantLLMBase": 1.5804959672112655,
53-
"utter-project/EuroLLM-9B": 1.734124223680335,
54-
"OpenLLM-France/Lucie-7B": 1.5905920397074582,
55-
"CohereForAI/c4ai-command-r-plus": 1.586136231175619,
56-
"allenai/OLMo-2-1124-7B-Instruct": 1.9298413832434025
83+
"google/gemma-7b": 1.6441557464889356,
84+
"gpt-3.5-turbo": 1.8529395175694223,
85+
"gpt-4": 1.8529395175694223,
86+
"inceptionai/jais-adapted-7b-chat": 1.9966221085813483,
87+
"meta-llama/Llama-2-7b-hf": 1.996634642501269,
88+
"meta-llama/Meta-Llama-3-8B-Instruct": 1.8492545451127114,
89+
"microsoft/Phi-3-medium-4k-instruct": 1.996634642501269,
90+
"microsoft/phi-2": 2.2364774671454626,
91+
"mistralai/Mistral-7B-Instruct-v0.3": 2.0805805711707306,
92+
"mistralai/Mistral-7B-v0.1": 2.0805805711707306,
93+
"nvidia/parakeet-ctc-1.1b": 3.140442572712403,
94+
"openai/whisper-large-v3": 1.9349677564910037,
95+
"tiiuae/falcon-7b": 1.8743913215138468,
96+
"utter-project/EuroLLM-9B": 1.734124223680335
5797
},
5898
"wikimedia/wikipedia/20231101.it": {
59-
"gpt-4": 2.0160554015873338,
60-
"google/gemma-7b": 1.7640779775993094,
61-
"microsoft/Phi-3-medium-4k-instruct": 2.155867537879554,
62-
"mistralai/Mistral-7B-v0.1": 2.2668730065610694,
63-
"meta-llama/Meta-Llama-3-8B-Instruct": 2.0103304809877343,
99+
"CohereForAI/aya-expanse-8b": 1.6885279064592478,
100+
"CohereForAI/c4ai-command-r-plus": 1.6885279064592478,
101+
"HuggingFaceH4/zephyr-7b-beta": 2.2668730065610694,
102+
"OpenLLM-France/Lucie-7B": 1.8978887248059664,
64103
"Qwen/Qwen2.5-7B": 2.0686855609275856,
104+
"allenai/OLMo-2-1124-7B-Instruct": 2.074444197196205,
65105
"bigscience/bloom-7b1": 1.9323933404810552,
66106
"croissantllm/CroissantLLMBase": 2.366233083163069,
67-
"utter-project/EuroLLM-9B": 1.7834779735534292,
68-
"OpenLLM-France/Lucie-7B": 1.8978887248059664,
69-
"CohereForAI/c4ai-command-r-plus": 1.6885279064592478,
70-
"allenai/OLMo-2-1124-7B-Instruct": 2.074444197196205
107+
"google/gemma-7b": 1.7640779775993094,
108+
"gpt-3.5-turbo": 2.0160554015873338,
109+
"gpt-4": 2.0160554015873338,
110+
"inceptionai/jais-adapted-7b-chat": 2.155867537879554,
111+
"meta-llama/Llama-2-7b-hf": 2.155867537879554,
112+
"meta-llama/Meta-Llama-3-8B-Instruct": 2.0103304809877343,
113+
"microsoft/Phi-3-medium-4k-instruct": 2.155867537879554,
114+
"microsoft/phi-2": 2.392288552181741,
115+
"mistralai/Mistral-7B-Instruct-v0.3": 2.2668730065610694,
116+
"mistralai/Mistral-7B-v0.1": 2.2668730065610694,
117+
"nvidia/parakeet-ctc-1.1b": 3.3884517090472626,
118+
"openai/whisper-large-v3": 2.1480724752021256,
119+
"tiiuae/falcon-7b": 2.005043864085395,
120+
"utter-project/EuroLLM-9B": 1.7834779735534292
71121
}
72122
}

0 commit comments

Comments
 (0)