Release/2.6.0

langtest/augmentation/debias.py

@@ -388,3 +388,16 @@ def get_ollama(
         return output_schema.model_validate_json(
             response.get("message", {}).get("content")
         )
+
+    def enhance_text(
+        self, text: str, bias_tolerance_level: Literal[1, 2, 3, 4, 5] = 2
+    ) -> str:
+        """Enhance the text by debiasing it."""
+        category, sub_category, rationale, rating, steps = self.detect_bias(text)
+        if rating and rating >= bias_tolerance_level:
+            debiased_text = self.debias_text(
+                text, category, sub_category, rationale, steps
+            )
+        else:
+            debiased_text = ""
+        return debiased_text

langtest/langtest.py

@@ -167,6 +167,7 @@ def __init__(
             self.hub = hub
             self._actual_model = model
         else:
+            additional_info = {}
             hub = None
             model_type = None
 
@@ -215,26 +216,40 @@ def __init__(
         if self.__prompt_config:
             self.prompt_manager = PromptManager.from_prompt_configs(self.__prompt_config)
 
-        # additional model info
-        additional_info = {
-            **(additional_info or {}),
-            **self._config.get("model_parameters", {}),
-        }
-
         # model section
         if isinstance(model, list):
             model_dict = {}
             for i in model:
                 model = i["model"]
                 hub = i["hub"]
 
+                # additional info for each model
+                additional_info = (
+                    {k: v for k, v in i.items() if k not in {"hub", "model", "type"}}
+                    if i
+                    else {}
+                )
+
+                # get config from model_parameters
+                model_params = self._config.get("model_parameters", {})
+                if isinstance(model_params, dict) and model in model_params:
+                    # additional info for each model
+                    additional_info = {**model_params[model], **additional_info}
+                else:
+                    # same additional info for all models
+                    additional_info = {**additional_info, **model_params}
+
                 model_dict[model] = self.task.model(
                     model, hub, model_type, **additional_info
                 )
 
                 self.model = model_dict
 
         else:
+            additional_info = {
+                **(additional_info or {}),
+                **self._config.get("model_parameters", {}),
+            }
             self.model = self.task.model(model, hub, model_type, **additional_info)
         # end model selection
         formatted_config = json.dumps(self._config, indent=1)

langtest/transform/clinical.py

@@ -599,6 +599,7 @@ def __drug_swap(self, result: str, text: str) -> str:
 class FCT(BaseClincial):
     """
     FCT class for the clinical tests
+    False Confidence Test
     """
 
     alias_name = "fct"
@@ -697,6 +698,7 @@ def transform(sample_list: List[Sample], *args, **kwargs):
             ):
                 # split by any letter. or number. or ) by re
                 options = re.sub(rf"{true_answer[3:]}", "None of the above", options)
+
             elif (
                 true_answer in options
                 and isinstance(options, list)
@@ -708,7 +710,16 @@ def transform(sample_list: List[Sample], *args, **kwargs):
                 ]
             sample.options = options
 
-            sample.expected_results = "None of the above"
+            # extract the [*]. from the true answer one character and .
+            option_pos = re.search(r"\b(?:[A-Za-z]|[0-9]|[IVXLCDM]+)[\.\)]", true_answer)
+            option_pos.end()
+            if option_pos:
+                option_pos = option_pos.end()
+                expected_results = f"{true_answer[:option_pos]} None of the above"
+            else:
+                expected_results = "None of the above"
+
+            sample.expected_results = expected_results
             sample.perturbed_context = ""
             sample.perturbed_question = ""
             transformed_samples.append(sample)
@@ -759,6 +770,7 @@ def transform(sample_list: List[Sample], *args, **kwargs):
 
         for sample in sample_list:
             sample.category = "clinical"
+            sample.test_type = "fqt"
             if (
                 sample.original_question is None
                 or sample.original_context is None
@@ -767,8 +779,12 @@ def transform(sample_list: List[Sample], *args, **kwargs):
                 continue
             if isinstance(sample, QASample):
                 selected = random.choice(questions)
+                if selected == sample.original_question:
+                    selected = random.choice(questions)
                 sample.original_question = selected
-                sample.expected_results = "None of the above"
+                sample.expected_results = kwargs.get(
+                    "expected_results", "None of the above"
+                )
                 sample.perturbed_context = ""
                 sample.perturbed_question = ""
 

pyproject.toml

@@ -76,7 +76,7 @@ matplotlib = {version = "^3.7.2", optional = true}
 tenacity = {version = "^8.2.2", optional = true}
 langchain = {version = "^0.3.15", optional = true}
 typing-extensions = "^4.10.0"
-langchain-openai = {version = "0.2.5", optional = true}
+langchain-openai = {version = "^0.3.7", optional = true}
 boto3 = {version = "^1.34.93", optional = true}
 importlib-resources = "^6.4.0"
 click = ">=8.1.7"