Minor bug fixes and updates to test and evaluator routes

pkiv · Jun 6, 2024 · 55b7dce · 55b7dce
1 parent b304cba
commit 55b7dce
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/test/evaluators.py b/test/evaluators.py
@@ -55,6 +55,7 @@ async def __call__(
 
         score = 1.0
         for approach, value in task_config["eval"]["reference_answers"].items():
+
             match approach:
                 case "exact_match":
                     logger.info(f"Evaluating exact_match for answer: Predicted: {pred} , Reference: {value}")
@@ -131,6 +132,7 @@ async def __call__(
         def clean_url(url: str) -> str:
             url = str(url)
             url = url.rstrip("/")
+            url = url.lower()
             return url
 
         def parse_url(url: str) -> tuple[str, dict[str, list[str]]]:

diff --git a/test/tasks/test.json b/test/tasks/test.json
@@ -512,9 +512,13 @@
                 "string_match"
             ],
             "reference_answers": {
-                "must_include": [
-                    "13.24 miles"
-                ]
+                "some_matches": [
+                    "13.24 miles",
+                    "10 miles",
+                    "approximately 10 miles",
+                    "10.32 miles"
+                ],
+                "min_required": 1
             },
             "reference_url": "",
             "program_html": [],