Skip to content

Add xnli ar #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 21, 2022
Merged

Add xnli ar #33

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d1f16cf
Accelerate `get_infos` by caching the `DataseInfoDict`s (#778)
VictorSanh May 22, 2022
6b1560f
fix `filter_english_datasets` since `languages` became `language` in …
VictorSanh Jul 3, 2022
ab6ad7e
fix empty documents - multi_news (#793)
VictorSanh Jul 3, 2022
0cc4b0c
Language tags (#771)
stephenbach Jul 8, 2022
1fdcbea
add arabic prompts
KhalidAlt Aug 16, 2022
64ac829
add arabic prompts and fix unicode issues
KhalidAlt Aug 16, 2022
148344d
fix semantic errors
KhalidAlt Aug 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 207 additions & 1 deletion promptsource/templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,212 @@
"zu": "Zulu",
}

# These are the metrics with which templates can be tagged
METRICS = {
"BLEU",
"ROUGE",
"Squad",
"Trivia QA",
"Accuracy",
"Pearson Correlation",
"Spearman Correlation",
"MultiRC",
"AUC",
"COQA F1",
"Edit Distance",
"Mean Reciprocal Rank",
"Other",
}

# These are the languages with which templates can be tagged. Keys are ISO 639-1
# tags, which are the actual tags we use. Values are English names shown in the
# UI for convenience.
LANGUAGES = {
"ab": "Abkhazian",
"aa": "Afar",
"af": "Afrikaans",
"ak": "Akan",
"sq": "Albanian",
"am": "Amharic",
"ar": "Arabic",
"an": "Aragonese",
"hy": "Armenian",
"as": "Assamese",
"av": "Avaric",
"ae": "Avestan",
"ay": "Aymara",
"az": "Azerbaijani",
"bm": "Bambara",
"ba": "Bashkir",
"eu": "Basque",
"be": "Belarusian",
"bn": "Bengali",
"bi": "Bislama",
"bs": "Bosnian",
"br": "Breton",
"bg": "Bulgarian",
"my": "Burmese",
"ca": "Catalan, Valencian",
"ch": "Chamorro",
"ce": "Chechen",
"ny": "Chichewa, Chewa, Nyanja",
"zh": "Chinese",
"cu": "Church Slavic, Old Slavonic, Church Slavonic, Old Bulgarian, Old Church Slavonic",
"cv": "Chuvash",
"kw": "Cornish",
"co": "Corsican",
"cr": "Cree",
"hr": "Croatian",
"cs": "Czech",
"da": "Danish",
"dv": "Divehi, Dhivehi, Maldivian",
"nl": "Dutch, Flemish",
"dz": "Dzongkha",
"en": "English",
"eo": "Esperanto",
"et": "Estonian",
"ee": "Ewe",
"fo": "Faroese",
"fj": "Fijian",
"fi": "Finnish",
"fr": "French",
"fy": "Western Frisian",
"ff": "Fulah",
"gd": "Gaelic, Scottish Gaelic",
"gl": "Galician",
"lg": "Ganda",
"ka": "Georgian",
"de": "German",
"el": "Greek, Modern (1453–)",
"kl": "Kalaallisut, Greenlandic",
"gn": "Guarani",
"gu": "Gujarati",
"ht": "Haitian, Haitian Creole",
"ha": "Hausa",
"he": "Hebrew",
"hz": "Herero",
"hi": "Hindi",
"ho": "Hiri Motu",
"hu": "Hungarian",
"is": "Icelandic",
"io": "Ido",
"ig": "Igbo",
"id": "Indonesian",
"ia": "Interlingua (International Auxiliary Language Association)",
"ie": "Interlingue, Occidental",
"iu": "Inuktitut",
"ik": "Inupiaq",
"ga": "Irish",
"it": "Italian",
"ja": "Japanese",
"jv": "Javanese",
"kn": "Kannada",
"kr": "Kanuri",
"ks": "Kashmiri",
"kk": "Kazakh",
"km": "Central Khmer",
"ki": "Kikuyu, Gikuyu",
"rw": "Kinyarwanda",
"ky": "Kirghiz, Kyrgyz",
"kv": "Komi",
"kg": "Kongo",
"ko": "Korean",
"kj": "Kuanyama, Kwanyama",
"ku": "Kurdish",
"lo": "Lao",
"la": "Latin",
"lv": "Latvian",
"li": "Limburgan, Limburger, Limburgish",
"ln": "Lingala",
"lt": "Lithuanian",
"lu": "Luba-Katanga",
"lb": "Luxembourgish, Letzeburgesch",
"mk": "Macedonian",
"mg": "Malagasy",
"ms": "Malay",
"ml": "Malayalam",
"mt": "Maltese",
"gv": "Manx",
"mi": "Maori",
"mr": "Marathi",
"mh": "Marshallese",
"mn": "Mongolian",
"na": "Nauru",
"nv": "Navajo, Navaho",
"nd": "North Ndebele",
"nr": "South Ndebele",
"ng": "Ndonga",
"ne": "Nepali",
"no": "Norwegian",
"nb": "Norwegian Bokmål",
"nn": "Norwegian Nynorsk",
"ii": "Sichuan Yi, Nuosu",
"oc": "Occitan",
"oj": "Ojibwa",
"or": "Oriya",
"om": "Oromo",
"os": "Ossetian, Ossetic",
"pi": "Pali",
"ps": "Pashto, Pushto",
"fa": "Persian",
"pl": "Polish",
"pt": "Portuguese",
"pa": "Punjabi, Panjabi",
"qu": "Quechua",
"ro": "Romanian, Moldavian, Moldovan",
"rm": "Romansh",
"rn": "Rundi",
"ru": "Russian",
"se": "Northern Sami",
"sm": "Samoan",
"sg": "Sango",
"sa": "Sanskrit",
"sc": "Sardinian",
"sr": "Serbian",
"sn": "Shona",
"sd": "Sindhi",
"si": "Sinhala, Sinhalese",
"sk": "Slovak",
"sl": "Slovenian",
"so": "Somali",
"st": "Southern Sotho",
"es": "Spanish, Castilian",
"su": "Sundanese",
"sw": "Swahili",
"ss": "Swati",
"sv": "Swedish",
"tl": "Tagalog",
"ty": "Tahitian",
"tg": "Tajik",
"ta": "Tamil",
"tt": "Tatar",
"te": "Telugu",
"th": "Thai",
"bo": "Tibetan",
"ti": "Tigrinya",
"to": "Tonga (Tonga Islands)",
"ts": "Tsonga",
"tn": "Tswana",
"tr": "Turkish",
"tk": "Turkmen",
"tw": "Twi",
"ug": "Uighur, Uyghur",
"uk": "Ukrainian",
"ur": "Urdu",
"uz": "Uzbek",
"ve": "Venda",
"vi": "Vietnamese",
"vo": "Volapük",
"wa": "Walloon",
"cy": "Welsh",
"wo": "Wolof",
"xh": "Xhosa",
"yi": "Yiddish",
"yo": "Yoruba",
"za": "Zhuang, Chuang",
"zu": "Zulu",
}


def highlight(input):
return "<span style='color: #F08080'>" + input + "</span>"
Expand Down Expand Up @@ -617,7 +823,7 @@ def write_to_file(self) -> None:
# We only create the folder if a template is written
if not os.path.exists(self.folder_path):
os.makedirs(self.folder_path)
yaml.dump(self.format_for_dump(), open(self.yaml_path, "w"))
yaml.dump(self.format_for_dump(), open(self.yaml_path, "w"),allow_unicode=True)

def add_template(self, template: "Template") -> None:
"""
Expand Down
118 changes: 80 additions & 38 deletions promptsource/templates/xnli/ar/templates.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,45 @@
dataset: xnli
subset: ar
templates:
97d2f061-06b0-4aa3-af53-871a2b06748f: !Template
answer_choices: صح ||| غير ذلك ||| خطأ
id: 97d2f061-06b0-4aa3-af53-871a2b06748f
jinja: '{{premise}}
سؤال:
{{hypothesis}}
صح ، خطأ إو غير ذلك
||| {{ answer_choices[label]
16836edd-898d-4508-92c0-e683f34aa41e: !Template
answer_choices: نعم ||| ربما ||| لا
id: 16836edd-898d-4508-92c0-e683f34aa41e
jinja: '{{premise}} بناء على النص السابق هل يصح القول بأن "{{hypothesis}}"? أجب
بنعم أو لا أو ربما. ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- ar
metrics:
- Accuracy
original_task: true
name: based on the previous passage
reference: ''
45e348b8-1dc4-485c-a182-6277069fe5a7: !Template
answer_choices: صحيحة ||| لا يمكن الجزم ||| خاطئة
id: 45e348b8-1dc4-485c-a182-6277069fe5a7
jinja: '{{premise}} بناء على المعلومات السابقة, هل الدعوى التالية: "{{hypothesis}}"
{{"صحيحة"}} أو {{"خاطئة"}} أو {{"لا يمكن الجزم"}}؟ ||| {{ answer_choices[label]
}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
- ar
metrics:
- Accuracy
original_task: true
name: GPT-3 style
reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there
is no task identifying tokens like "anli R1: ".'
name: claim true/false/inconclusive
reference: Sanh et al. 2021
8b75ff67-bb66-413b-a33d-325707b035d7: !Template
answer_choices: نعم ||| ربما ||| لا
id: 8b75ff67-bb66-413b-a33d-325707b035d7
jinja: "افترض
{{premise}}
هل نستطيح أن نستنتج أن
"{{hypothesis}}"
نعم ، لا أو ربما؟
||| {{ answer_choices[label] }} "

jinja: 'اذا كان لدينا المقدمة التالية {{premise}} هل يمكن أن نتوصل إلى الفرضية
التالية "{{hypothesis}}"؟ أجب بـ نعم أو لا أو ربما؟ ||| {{ answer_choices[label]
}} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
- ar
metrics:
- Accuracy
original_task: true
Expand All @@ -42,33 +48,56 @@ templates:
8bda8e36-c881-4c9a-a3a9-eec68388a6f6: !Template
answer_choices: نعم ||| ربما ||| لا
id: 8bda8e36-c881-4c9a-a3a9-eec68388a6f6
jinja:"{{premise}}
هل يمكننا استنتاج
"{{hypothesis}}"
نعم، لا أو ربما؟ ||| {{ answer_choices[label] }} "
jinja: '{{premise}} هل يمكن القول بالفرضية التالية "{{hypothesis}}" بناء على
ما سبق؟ أجب بنعم أو ﻻ أو ربما ؟ ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
- ar
metrics:
- Accuracy
original_task: true
name: justified in saying
reference: Webson & Pavlick 2021
97d2f061-06b0-4aa3-af53-871a2b06748f: !Template
answer_choices: صح ||| غير ذلك ||| خطأ
id: 97d2f061-06b0-4aa3-af53-871a2b06748f
jinja: '{{premise}} سؤال: {{hypothesis}} صح ، خطأ إو غير ذلك ||| {{ answer_choices[label]
}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- ar
metrics:
- Accuracy
original_task: true
name: GPT-3 style
reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there
is no task identifying tokens like "anli R1: ".'
ba194be7-4818-4e24-b5f9-bbcccaa4e166: !Template
answer_choices: نعم ||| ربما ||| لا
id: ba194be7-4818-4e24-b5f9-bbcccaa4e166
jinja: 'بناء على المقدمة التالية {{premise}} هل يمكن أن تليها الفرضة التالية:
{{hypothesis}} أجب بـ نعم أو ربما أو لا ؟ ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- ar
metrics:
- Accuracy
original_task: true
name: does it follow that
reference: Sanh et al. 2021
e44c2947-f8e0-49ea-9770-e59f0581a921: !Template
answer_choices: أكيدة ||| ممكنة ||| مستحيلة
id: e44c2947-f8e0-49ea-9770-e59f0581a921
jinja: "افترض أن
{{premise}}
لذلك فإن
\"{{hypothesis}}\"
تعتبر
{{\"أكيدة\"}}, {{\"ممكنة\"}}, or {{\"مستحيلة\"}}?
||| {{ answer_choices[label] }}"
jinja: "اذا كانت لدينا المقدمة الصحيحة التالية {{premise}} \nفإن, \"{{hypothesis}}\"\
\ هي دعوى {{\"مؤكدة\"}} أو {{\"ممكنة\"}} أو {{\"مستحيلة\"}}؟ ||| {{ answer_choices[label]\
\ }}\n"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
- ar
metrics:
- Accuracy
original_task: true
Expand All @@ -77,16 +106,29 @@ templates:
ea368462-3a66-4222-9de1-05d66037a708: !Template
answer_choices: صحيحة ||| لا يمكن التقرير ||| غير صحيحة
id: ea368462-3a66-4222-9de1-05d66037a708
jinja: '{{premise}}
باستخدام الوصف الذي في الأعلى وكذلك المعرفة العامة،
"{{hypothesis}}"
تعتبر صحيحة، غير صحيحة، لا يمكن التقرير؟ ||| {{ answer_choices[label] }}'
jinja: " {{premise}}\n بإستخدام الوصف بالأعلى وحسب ما تعرفه من المعرفة العامة\
\ فقط هل يمكن القول بأن الفرضية التالية: \"{{hypothesis}}\" بلا شك صحيحة أو\
\ خاطئة أو لا يمكن التقرير؟ ||| {{ answer_choices[label] }}\n"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
- ar
metrics:
- Accuracy
original_task: true
name: MNLI crowdsource
reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers.
ef61016d-6451-4462-b69c-cb2ac7856b32: !Template
answer_choices: صحيحة ||| لا يمكن الجزم ||| خاطئة
id: ef61016d-6451-4462-b69c-cb2ac7856b32
jinja: 'اذا كانت المقدمة التالية صحيحة: {{premise}} فإن الفرضية التالية: "{{hypothesis}}" {{"صحيحة"}}
أو {{"خاطئة"}} أو {{"لايمكن الجزم"}}؟ ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- ar
metrics:
- Accuracy
original_task: true
name: take the following as truth
reference: Sanh et al. 2021
2 changes: 1 addition & 1 deletion promptsource/templates/xnli/es/templates.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
dataset: xnli
subset: vi
subset: es
templates:
37d2f061-06b0-4aa3-af53-871a2b06748f: !Template
answer_choices: True ||| Neither ||| False
Expand Down