Skip to content

Commit 3bb8a50

Browse files
authored
Fix task metric type mismatch (#743)
* fixing task metric type * fix style
1 parent 1607dc1 commit 3bb8a50

File tree

1 file changed

+51
-9
lines changed

1 file changed

+51
-9
lines changed

src/lighteval/tasks/default_tasks.py

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6727,7 +6727,14 @@
67276727
few_shots_split=None,
67286728
few_shots_select=None,
67296729
generation_size=128,
6730-
metric=[Metrics.exact_match, Metrics.quasi_exact_match, Metrics.f1_score, Metrics.rougeL, "bleu_1", "bleu_4"],
6730+
metric=[
6731+
Metrics.exact_match,
6732+
Metrics.quasi_exact_match,
6733+
Metrics.f1_score,
6734+
Metrics.rougeL,
6735+
Metrics.bleu_1,
6736+
Metrics.bleu_4,
6737+
],
67316738
stop_sequence=["\n"],
67326739
trust_dataset=True,
67336740
version=0,
@@ -7752,7 +7759,7 @@
77527759
few_shots_split=None,
77537760
few_shots_select=None,
77547761
generation_size=1,
7755-
metric=[Metrics.loglikelihood_acc, "loglikelihood_f1"],
7762+
metric=[Metrics.loglikelihood_acc, Metrics.loglikelihood_f1],
77567763
stop_sequence=["\n"],
77577764
trust_dataset=True,
77587765
version=0,
@@ -7784,7 +7791,7 @@
77847791
few_shots_split=None,
77857792
few_shots_select=None,
77867793
generation_size=1,
7787-
metric=[Metrics.loglikelihood_acc, "loglikelihood_f1"],
7794+
metric=[Metrics.loglikelihood_acc, Metrics.loglikelihood_f1],
77887795
stop_sequence=["\n"],
77897796
trust_dataset=True,
77907797
version=0,
@@ -10103,7 +10110,14 @@
1010310110
few_shots_split=None,
1010410111
few_shots_select=None,
1010510112
generation_size=128,
10106-
metric=[Metrics.exact_match, Metrics.quasi_exact_match, Metrics.f1_score, Metrics.rougeL, "bleu_1", "bleu_4"],
10113+
metric=[
10114+
Metrics.exact_match,
10115+
Metrics.quasi_exact_match,
10116+
Metrics.f1_score,
10117+
Metrics.rougeL,
10118+
Metrics.bleu_1,
10119+
Metrics.bleu_4,
10120+
],
1010710121
stop_sequence=["\n"],
1010810122
trust_dataset=True,
1010910123
version=0,
@@ -10119,7 +10133,14 @@
1011910133
few_shots_split=None,
1012010134
few_shots_select=None,
1012110135
generation_size=128,
10122-
metric=[Metrics.exact_match, Metrics.quasi_exact_match, Metrics.f1_score, Metrics.rougeL, "bleu_1", "bleu_4"],
10136+
metric=[
10137+
Metrics.exact_match,
10138+
Metrics.quasi_exact_match,
10139+
Metrics.f1_score,
10140+
Metrics.rougeL,
10141+
Metrics.bleu_1,
10142+
Metrics.bleu_4,
10143+
],
1012310144
stop_sequence=["\n"],
1012410145
trust_dataset=True,
1012510146
version=0,
@@ -10135,7 +10156,14 @@
1013510156
few_shots_split=None,
1013610157
few_shots_select=None,
1013710158
generation_size=128,
10138-
metric=[Metrics.exact_match, Metrics.quasi_exact_match, Metrics.f1_score, Metrics.rougeL, "bleu_1", "bleu_4"],
10159+
metric=[
10160+
Metrics.exact_match,
10161+
Metrics.quasi_exact_match,
10162+
Metrics.f1_score,
10163+
Metrics.rougeL,
10164+
Metrics.bleu_1,
10165+
Metrics.bleu_4,
10166+
],
1013910167
stop_sequence=["\n"],
1014010168
trust_dataset=True,
1014110169
version=0,
@@ -10173,7 +10201,14 @@
1017310201
few_shots_split=None,
1017410202
few_shots_select=None,
1017510203
generation_size=512,
10176-
metric=[Metrics.exact_match, Metrics.quasi_exact_match, Metrics.f1_score, Metrics.rougeL, "bleu_1", "bleu_4"],
10204+
metric=[
10205+
Metrics.exact_match,
10206+
Metrics.quasi_exact_match,
10207+
Metrics.f1_score,
10208+
Metrics.rougeL,
10209+
Metrics.bleu_1,
10210+
Metrics.bleu_4,
10211+
],
1017710212
stop_sequence=["\n"],
1017810213
trust_dataset=True,
1017910214
version=0,
@@ -13744,7 +13779,14 @@
1374413779
few_shots_split=None,
1374513780
few_shots_select=None,
1374613781
generation_size=100,
13747-
metric=[Metrics.exact_match, Metrics.quasi_exact_match, Metrics.f1_score, Metrics.rougeL, "bleu_1", "bleu_4"],
13782+
metric=[
13783+
Metrics.exact_match,
13784+
Metrics.quasi_exact_match,
13785+
Metrics.f1_score,
13786+
Metrics.rougeL,
13787+
Metrics.bleu_1,
13788+
Metrics.bleu_4,
13789+
],
1374813790
stop_sequence=["\n"],
1374913791
trust_dataset=True,
1375013792
version=0,
@@ -15407,7 +15449,7 @@
1540715449
few_shots_split=None,
1540815450
few_shots_select=None,
1540915451
generation_size=1,
15410-
metric=[Metrics.loglikelihood_acc, "multi_f1_numeric"],
15452+
metric=[Metrics.loglikelihood_acc, Metrics.multi_f1_numeric],
1541115453
stop_sequence=["\n"],
1541215454
trust_dataset=True,
1541315455
version=0,

0 commit comments

Comments
 (0)