initial hnsw cached beir scores.

src/main/python/run_regression.py

@@ -418,22 +418,25 @@ def evaluate_and_verify(yaml_data, dry_run):
                     #         tolerance_ok = dl20_flat_tolerance[model_type][model['name']]
 
                 if using_hnsw:
-                    # Extract model
-                    match = hnsw_model_type_pattern.search(model['name'])
-                    model_type = match.group(1)
-
-                    if 'BEIR' in topic_set['name']:
-                        # Extract BEIR dataset
-                        match = beir_dataset_pattern.search(topic_set['name'])
-                        beir_dataset = match.group(1)
-
-                        tolerance_ok = beir_hnsw_tolerance[model_type][beir_dataset]
-                    elif 'MS MARCO Passage' in topic_set['name']:
-                        tolerance_ok = msmarco_v1_hnsw_tolerance[model_type][model['name']]
-                    elif 'DL19' in topic_set['name']:
-                        tolerance_ok = dl19_hnsw_tolerance[model_type][model['name']]
-                    elif 'DL20' in topic_set['name']:
-                        tolerance_ok = dl20_hnsw_tolerance[model_type][model['name']]
+                    if 'tolerance' in model:
+                        tolerance_ok = model['tolerance'][metric['metric']][i]
+                    else:
+                        # Extract model
+                        match = hnsw_model_type_pattern.search(model['name'])
+                        model_type = match.group(1)
+
+                        if 'BEIR' in topic_set['name']:
+                            # Extract BEIR dataset
+                            match = beir_dataset_pattern.search(topic_set['name'])
+                            beir_dataset = match.group(1)
+
+                            tolerance_ok = beir_hnsw_tolerance[model_type][beir_dataset]
+                        elif 'MS MARCO Passage' in topic_set['name']:
+                            tolerance_ok = msmarco_v1_hnsw_tolerance[model_type][model['name']]
+                        elif 'DL19' in topic_set['name']:
+                            tolerance_ok = dl19_hnsw_tolerance[model_type][model['name']]
+                        elif 'DL20' in topic_set['name']:
+                            tolerance_ok = dl20_hnsw_tolerance[model_type][model['name']]
 
                 if using_flat or using_hnsw:
                     result_str = (f'expected: {expected:.4f} actual: {actual:.4f} '

src/main/resources/regression/beir-v1.0.0-arguana.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9915
       R@1000:
         - 0.9964
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-bioasq.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6317
       R@1000:
         - 0.8059
+    tolerance:
+      nDCG@10:
+        - 0.02
+      R@100:
+        - 0.03
+      R@1000:
+        - 0.04

src/main/resources/regression/beir-v1.0.0-climate-fever.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6362
       R@1000:
         - 0.8307
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.003

src/main/resources/regression/beir-v1.0.0-cqadupstack-android.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.8454
       R@1000:
         - 0.9611
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-cqadupstack-english.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7587
       R@1000:
         - 0.8839
+    tolerance:
+      nDCG@10:
+        - 0.002
+      R@100:
+        - 0.003
+      R@1000:
+        - 0.003

src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9036
       R@1000:
         - 0.9719
+    tolerance:
+      nDCG@10:
+        - 0.002
+      R@100:
+        - 0.003
+      R@1000:
+        - 0.003

src/main/resources/regression/beir-v1.0.0-cqadupstack-gis.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7682
       R@1000:
         - 0.9117
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.002
+      R@1000:
+        - 0.002

src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6922
       R@1000:
         - 0.8810
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-cqadupstack-physics.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.8081
       R@1000:
         - 0.9406
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7856
       R@1000:
         - 0.9348
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-cqadupstack-stats.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6727
       R@1000:
         - 0.8445
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.005

src/main/resources/regression/beir-v1.0.0-cqadupstack-tex.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6486
       R@1000:
         - 0.8537
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.002
+      R@1000:
+        - 0.002

src/main/resources/regression/beir-v1.0.0-cqadupstack-unix.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7797
       R@1000:
         - 0.9237
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.002

src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7774
       R@1000:
         - 0.9380
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.002

src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7065
       R@1000:
         - 0.8861
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.002
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-dbpedia-entity.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.5303
       R@1000:
         - 0.7833
+    tolerance:
+      nDCG@10:
+        - 0.004
+      R@100:
+        - 0.007
+      R@1000:
+        - 0.02

src/main/resources/regression/beir-v1.0.0-fever.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9719
       R@1000:
         - 0.9855
+    tolerance:
+      nDCG@10:
+        - 0.006
+      R@100:
+        - 0.01
+      R@1000:
+        - 0.01

src/main/resources/regression/beir-v1.0.0-fiqa.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7415
       R@1000:
         - 0.9083
+    tolerance:
+      nDCG@10:
+        - 0.002
+      R@100:
+        - 0.004
+      R@1000:
+        - 0.007

src/main/resources/regression/beir-v1.0.0-hotpotqa.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.8727
       R@1000:
         - 0.9424
+    tolerance:
+      nDCG@10:
+        - 0.02
+      R@100:
+        - 0.02
+      R@1000:
+        - 0.02

src/main/resources/regression/beir-v1.0.0-nfcorpus.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.3368
       R@1000:
         - 0.6622
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.002

src/main/resources/regression/beir-v1.0.0-nq.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9415
       R@1000:
         - 0.9859
+    tolerance:
+      nDCG@10:
+        - 0.003
+      R@100:
+        - 0.007
+      R@1000:
+        - 0.009

src/main/resources/regression/beir-v1.0.0-quora.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9967
       R@1000:
         - 0.9998
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-robust04.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.3507
       R@1000:
         - 0.5981
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.002
+      R@1000:
+        - 0.003

src/main/resources/regression/beir-v1.0.0-scidocs.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.4959
       R@1000:
         - 0.7824
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-scifact.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9667
       R@1000:
         - 0.9967
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-signal1m.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.3112
       R@1000:
         - 0.5331
+    tolerance:
+      nDCG@10:
+        - 0.006
+      R@100:
+        - 0.02
+      R@1000:
+        - 0.03

src/main/resources/regression/beir-v1.0.0-trec-covid.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.1406
       R@1000:
         - 0.4768
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-trec-news.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.4992
       R@1000:
         - 0.7875
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.002
+      R@1000:
+        - 0.007

src/main/resources/regression/beir-v1.0.0-webis-touche2020.bge-base-en-v1.5.hnsw.cached.yaml

@@ -51,3 +51,10 @@ models:
         - 0.4857
       R@1000:
         - 0.8298
+    tolerance:
+      nDCG@10:
+        - 0.001
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.001