Skip to content

Commit

Permalink
add vizwiz retrieval and results
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jamie-Stirling
Jamie-Stirling committed Oct 15, 2024
1 parent 411e313 commit 74a2400
Show file tree
Hide file tree
Showing 4 changed files with 289 additions and 0 deletions.
2 changes: 2 additions & 0 deletions mteb/tasks/Image/Any2AnyRetrieval/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
from .eng.TUBerlinT2IRetrieval import *
from .eng.VisualNewsI2TRetrieval import *
from .eng.VisualNewsT2IRetrieval import *
from .eng.VizWizIT2TRetrieval import *
from .eng.VQA2IT2TRetrieval import *
from .eng.WebQAT2ITRetrieval import *
from .eng.WebQAT2TRetrieval import *
from .multilingual.WITT2IRetrieval import *
Expand Down
50 changes: 50 additions & 0 deletions mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval
from mteb.abstasks.TaskMetadata import TaskMetadata


class VQA2IT2TRetrieval(AbsTaskAny2AnyRetrieval):
metadata = TaskMetadata(
name="VQA2IT2TRetrieval",
description="Retrieve the correct answer for a question about an image.",
reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Goyal_Making_the_v_CVPR_2017_paper.html",
dataset={
"path": "JamieSJS/vqa-2",
"revision": "addc919d80589a767212ca455657f4e6c55e71f8",
"trust_remote_code": True,
},
type="Retrieval",
category="it2t",
eval_splits=["test"],
eval_langs=["eng-Latn"],
main_score="ndcg_at_10",
date=("2017-07-01", "2017-07-01"),
domains=["Web"],
task_subtypes=["Image Text Retrieval"],
license="CC BY-SA 4.0",
annotations_creators="derived",
dialect=[],
modalities=["text", "image"],
sample_creation="found",
bibtex_citation="""@InProceedings{Goyal_2017_CVPR,
author = {Goyal, Yash and Khot, Tejas and Summers-Stay, Douglas and Batra, Dhruv and Parikh, Devi},
title = {Making the v in VQA Matter: Elevating the Role of Image Understanding in Visual Question Answering},
booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
month = {July},
year = {2017}
}
""",
descriptive_stats={
"n_samples": {"test": 214354},
"avg_character_length": {
"test": {
"average_document_length": 0.0,
"average_query_length": 0.0,
"num_documents": 2143540,
"num_queries": 214354,
"average_relevant_docs_per_query": 10,
}
},
},
)
51 changes: 51 additions & 0 deletions mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
from __future__ import annotations

from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval
from mteb.abstasks.TaskMetadata import TaskMetadata


class VizWizIT2TRetrieval(AbsTaskAny2AnyRetrieval):
metadata = TaskMetadata(
name="VizWizIT2TRetrieval",
description="Retrieve the correct answer for a question about an image.",
reference="https://openaccess.thecvf.com/content_cvpr_2018/papers/Gurari_VizWiz_Grand_Challenge_CVPR_2018_paper.pdf",
dataset={
"path": "JamieSJS/vizwiz",
"revision": "723c6ffed034eb9dcbb44777e1a9ddf97c5ee28a",
"trust_remote_code": True,
},
type="Retrieval",
category="it2t",
eval_splits=["test"],
eval_langs=["eng-Latn"],
main_score="ndcg_at_10",
date=("2018-01-01", "2018-01-01"),
domains=["Web"],
task_subtypes=["Image Text Retrieval"],
license="CC BY-SA 4.0",
annotations_creators="derived",
dialect=[],
modalities=["text", "image"],
sample_creation="found",
bibtex_citation="""@inproceedings{gurari2018vizwiz,
title={Vizwiz grand challenge: Answering visual questions from blind people},
author={Gurari, Danna and Li, Qing and Stangl, Abigale J and Guo, Anhong and Lin, Chi and Grauman, Kristen and Luo, Jiebo and Bigham, Jeffrey P},
booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
pages={3608--3617},
year={2018}
}
""",
descriptive_stats={
"n_samples": {"test": 214354},
"avg_character_length": {
"test": {
"average_document_length": 0.0,
"average_query_length": 0.0,
"num_documents": 2143540,
"num_queries": 214354,
"average_relevant_docs_per_query": 10,
}
},
},
)
186 changes: 186 additions & 0 deletions ...__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
{
"dataset_revision": "723c6ffed034eb9dcbb44777e1a9ddf97c5ee28a",
"evaluation_time": 32.021708726882935,
"kg_co2_emissions": null,
"mteb_version": "1.12.90",
"scores": {
"test": [
{
"cv_recall_at_1": 0.0294,
"cv_recall_at_10": 0.07317,
"cv_recall_at_100": 0.19819,
"cv_recall_at_1000": 0.3737,
"cv_recall_at_20": 0.09863,
"cv_recall_at_3": 0.04608,
"cv_recall_at_5": 0.05788,
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.01511,
"map_at_1": 0.00301,
"map_at_10": 0.00749,
"map_at_100": 0.01081,
"map_at_1000": 0.01176,
"map_at_20": 0.0088,
"map_at_3": 0.00507,
"map_at_5": 0.00605,
"mrr_at_1": 0.029404954850659876,
"mrr_at_10": 0.041434029041114,
"mrr_at_100": 0.045365167700329743,
"mrr_at_1000": 0.04612327351299033,
"mrr_at_20": 0.043187763329325876,
"mrr_at_3": 0.036736898973527825,
"mrr_at_5": 0.03942270587327312,
"nauc_cv_recall_at_1000_diff1": 0.11813681061217512,
"nauc_cv_recall_at_1000_max": -0.11894128215838488,
"nauc_cv_recall_at_1000_std": 0.3782492530808885,
"nauc_cv_recall_at_100_diff1": 0.11427439308010998,
"nauc_cv_recall_at_100_max": -0.09531881188079029,
"nauc_cv_recall_at_100_std": 0.2069207353440963,
"nauc_cv_recall_at_10_diff1": 0.20419699826470403,
"nauc_cv_recall_at_10_max": -0.11315602683152005,
"nauc_cv_recall_at_10_std": 0.11079116202458081,
"nauc_cv_recall_at_1_diff1": 0.3780410690211136,
"nauc_cv_recall_at_1_max": -0.15996276050998642,
"nauc_cv_recall_at_1_std": 0.12084151368405678,
"nauc_cv_recall_at_20_diff1": 0.17482215533278464,
"nauc_cv_recall_at_20_max": -0.09614325077967463,
"nauc_cv_recall_at_20_std": 0.11622067910258244,
"nauc_cv_recall_at_3_diff1": 0.3021499883385322,
"nauc_cv_recall_at_3_max": -0.1409817803959759,
"nauc_cv_recall_at_3_std": 0.12190544061690316,
"nauc_cv_recall_at_5_diff1": 0.2422495719721318,
"nauc_cv_recall_at_5_max": -0.11604909641982115,
"nauc_cv_recall_at_5_std": 0.12525785371426437,
"nauc_map_at_1000_diff1": 0.16429877210946825,
"nauc_map_at_1000_max": -0.1295306253626044,
"nauc_map_at_1000_std": 0.2062946969445036,
"nauc_map_at_100_diff1": 0.1670600341719391,
"nauc_map_at_100_max": -0.12982814569101003,
"nauc_map_at_100_std": 0.18876332652966407,
"nauc_map_at_10_diff1": 0.19138314983144086,
"nauc_map_at_10_max": -0.12835016161145998,
"nauc_map_at_10_std": 0.15183799317664892,
"nauc_map_at_1_diff1": 0.3572077845896727,
"nauc_map_at_1_max": -0.1591774719332963,
"nauc_map_at_1_std": 0.1257863823021691,
"nauc_map_at_20_diff1": 0.17047854383467648,
"nauc_map_at_20_max": -0.12495405745802321,
"nauc_map_at_20_std": 0.16271914594752626,
"nauc_map_at_3_diff1": 0.25952802533398156,
"nauc_map_at_3_max": -0.1473725924476619,
"nauc_map_at_3_std": 0.13084611961877016,
"nauc_map_at_5_diff1": 0.2102820356246173,
"nauc_map_at_5_max": -0.13351667721655666,
"nauc_map_at_5_std": 0.14460103788615644,
"nauc_mrr_at_1000_diff1": 0.2882430552866343,
"nauc_mrr_at_1000_max": -0.13678972890948193,
"nauc_mrr_at_1000_std": 0.12898324685751192,
"nauc_mrr_at_100_diff1": 0.2891333801979083,
"nauc_mrr_at_100_max": -0.1371078360866261,
"nauc_mrr_at_100_std": 0.12577922767556043,
"nauc_mrr_at_10_diff1": 0.30678787633680227,
"nauc_mrr_at_10_max": -0.1417415793576384,
"nauc_mrr_at_10_std": 0.11840012286416857,
"nauc_mrr_at_1_diff1": 0.3780410690211136,
"nauc_mrr_at_1_max": -0.15996276050998642,
"nauc_mrr_at_1_std": 0.12084151368405678,
"nauc_mrr_at_20_diff1": 0.29802916652653827,
"nauc_mrr_at_20_max": -0.1380992977602512,
"nauc_mrr_at_20_std": 0.11913643771545089,
"nauc_mrr_at_3_diff1": 0.34218208940138206,
"nauc_mrr_at_3_max": -0.15259828173240922,
"nauc_mrr_at_3_std": 0.11980199564671674,
"nauc_mrr_at_5_diff1": 0.31906571381939547,
"nauc_mrr_at_5_max": -0.14353131092973306,
"nauc_mrr_at_5_std": 0.12118536859900152,
"nauc_ndcg_at_1000_diff1": 0.1382546228263889,
"nauc_ndcg_at_1000_max": -0.1186022001179963,
"nauc_ndcg_at_1000_std": 0.29134546582965304,
"nauc_ndcg_at_100_diff1": 0.1608084546142371,
"nauc_ndcg_at_100_max": -0.1208877559776067,
"nauc_ndcg_at_100_std": 0.20420457544308251,
"nauc_ndcg_at_10_diff1": 0.21347296273381702,
"nauc_ndcg_at_10_max": -0.12360631260617351,
"nauc_ndcg_at_10_std": 0.13835357292918368,
"nauc_ndcg_at_1_diff1": 0.35720778458967256,
"nauc_ndcg_at_1_max": -0.15917747193329645,
"nauc_ndcg_at_1_std": 0.12578638230216885,
"nauc_ndcg_at_20_diff1": 0.189575288463788,
"nauc_ndcg_at_20_max": -0.12047147115102609,
"nauc_ndcg_at_20_std": 0.14823959023714925,
"nauc_ndcg_at_3_diff1": 0.28072213819651337,
"nauc_ndcg_at_3_max": -0.1493321151811889,
"nauc_ndcg_at_3_std": 0.1313988281635346,
"nauc_ndcg_at_5_diff1": 0.23487221921337986,
"nauc_ndcg_at_5_max": -0.12854699731824779,
"nauc_ndcg_at_5_std": 0.13838213366410212,
"nauc_precision_at_1000_diff1": 0.09153318291899752,
"nauc_precision_at_1000_max": -0.10712625620860772,
"nauc_precision_at_1000_std": 0.3486808234128117,
"nauc_precision_at_100_diff1": 0.11829783638630895,
"nauc_precision_at_100_max": -0.11229273014278543,
"nauc_precision_at_100_std": 0.2457874398625524,
"nauc_precision_at_10_diff1": 0.1586640506586383,
"nauc_precision_at_10_max": -0.10658265337446211,
"nauc_precision_at_10_std": 0.1433984006213967,
"nauc_precision_at_1_diff1": 0.35720778458967256,
"nauc_precision_at_1_max": -0.15917747193329645,
"nauc_precision_at_1_std": 0.12578638230216885,
"nauc_precision_at_20_diff1": 0.13529505359357108,
"nauc_precision_at_20_max": -0.10702832193841087,
"nauc_precision_at_20_std": 0.16042770655667163,
"nauc_precision_at_3_diff1": 0.2468790893829598,
"nauc_precision_at_3_max": -0.1425487542554598,
"nauc_precision_at_3_std": 0.1347458330420782,
"nauc_precision_at_5_diff1": 0.17923534723711776,
"nauc_precision_at_5_max": -0.10942897722644009,
"nauc_precision_at_5_std": 0.1456878106837016,
"nauc_recall_at_1000_diff1": 0.09153318291899752,
"nauc_recall_at_1000_max": -0.1071262562086077,
"nauc_recall_at_1000_std": 0.34868082341281187,
"nauc_recall_at_100_diff1": 0.11829783638630909,
"nauc_recall_at_100_max": -0.11229273014278539,
"nauc_recall_at_100_std": 0.24578743986255244,
"nauc_recall_at_10_diff1": 0.1586640506586383,
"nauc_recall_at_10_max": -0.10658265337446211,
"nauc_recall_at_10_std": 0.1433984006213967,
"nauc_recall_at_1_diff1": 0.3572077845896727,
"nauc_recall_at_1_max": -0.1591774719332963,
"nauc_recall_at_1_std": 0.1257863823021691,
"nauc_recall_at_20_diff1": 0.13529505359357108,
"nauc_recall_at_20_max": -0.10702832193841087,
"nauc_recall_at_20_std": 0.16042770655667163,
"nauc_recall_at_3_diff1": 0.24687908938295963,
"nauc_recall_at_3_max": -0.14254875425545974,
"nauc_recall_at_3_std": 0.1347458330420781,
"nauc_recall_at_5_diff1": 0.17923534723711776,
"nauc_recall_at_5_max": -0.10942897722644009,
"nauc_recall_at_5_std": 0.1456878106837016,
"ndcg_at_1": 0.0301,
"ndcg_at_10": 0.01511,
"ndcg_at_100": 0.03019,
"ndcg_at_1000": 0.04943,
"ndcg_at_20": 0.01882,
"ndcg_at_3": 0.02242,
"ndcg_at_5": 0.01915,
"precision_at_1": 0.0301,
"precision_at_10": 0.01197,
"precision_at_100": 0.00477,
"precision_at_1000": 0.0012,
"precision_at_20": 0.00935,
"precision_at_3": 0.02022,
"precision_at_5": 0.01635,
"recall_at_1": 0.00301,
"recall_at_10": 0.01197,
"recall_at_100": 0.04767,
"recall_at_1000": 0.11975,
"recall_at_20": 0.01871,
"recall_at_3": 0.00607,
"recall_at_5": 0.00817
}
]
},
"task_name": "VizWizIT2TRetrieval"
}

0 comments on commit 74a2400

Please sign in to comment.