Skip to content

Commit

Permalink
add large model results
Browse files Browse the repository at this point in the history
  • Loading branch information
isaac-chung committed Oct 16, 2024
1 parent d2f6cc0 commit 9c0d836
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 1 deletion.
5 changes: 4 additions & 1 deletion mteb/models/moco_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@ def __init__(
):
self.model_name = model_name
self.device = device
model = timm.create_model('vit_base_patch16_224', pretrained=True, num_classes=0, pretrained_cfg_overlay=dict(hf_hub_id=model_name))
name = 'vit_base_patch16_224'
if 'vit-l' in model_name:
name = 'vit_large_patch16_224'
model = timm.create_model(name, pretrained=True, num_classes=0, pretrained_cfg_overlay=dict(hf_hub_id=model_name))

self.model = model.eval()

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
{
"dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b",
"evaluation_time": 780.7572722434998,
"kg_co2_emissions": null,
"mteb_version": "1.14.21",
"scores": {
"test": [
{
"accuracy": 0.8990196078431373,
"f1": 0.8976575858316652,
"f1_weighted": 0.8973426503552833,
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.8990196078431373,
"scores_per_experiment": [
{
"accuracy": 0.8960784313725491,
"f1": 0.8954744443526484,
"f1_weighted": 0.8949013011291541
},
{
"accuracy": 0.9009803921568628,
"f1": 0.89924527322374,
"f1_weighted": 0.89919388345273
},
{
"accuracy": 0.8960784313725491,
"f1": 0.8937528098526334,
"f1_weighted": 0.8936883759623736
},
{
"accuracy": 0.8990196078431373,
"f1": 0.8985132758082764,
"f1_weighted": 0.8977617544247366
},
{
"accuracy": 0.9029411764705882,
"f1": 0.901302125921028,
"f1_weighted": 0.901167936807422
}
]
}
]
},
"task_name": "OxfordFlowersClassification"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"name": "nyu-visionx/moco-v3-vit-l", "revision": "7bf75358d616f39b9716148bf4e3425f3bd35b47", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "mocov3_loader"}

0 comments on commit 9c0d836

Please sign in to comment.