feat: remove examples (#1708)

Author: jjmachan

src/ragas/metrics/_aspect_critic.py

@@ -118,13 +118,13 @@ def __init__(
             llm=llm,
         )
 
-        self.definition = definition
+        self._definition = definition
         self.single_turn_prompt = single_turn_prompt or SingleTurnAspectCriticPrompt()
         self.multi_turn_prompt = multi_turn_prompt or MultiTurnAspectCriticPrompt()
         self.max_retries = max_retries
 
         # update the instruction for the prompts with the definition
-        instruction = f"Evaluate the Input based on the criterial defined. Use only 'Yes' (1) and 'No' (0) as verdict.\nCriteria Definition: {self.definition}"
+        instruction = f"Evaluate the Input based on the criterial defined. Use only 'Yes' (1) and 'No' (0) as verdict.\nCriteria Definition: {self._definition}"
         self.single_turn_prompt.instruction = instruction
         self.multi_turn_prompt.instruction = instruction
 
@@ -135,7 +135,19 @@ def __init__(
         )
 
     def __repr__(self) -> str:
-        return f"{self.name}(definition='{self.definition}', required_columns={self.required_columns}, llm={self.llm})"
+        return f"{self.name}(definition='{self._definition}', required_columns={self.required_columns}, llm={self.llm})"
+
+    @property
+    def definition(self) -> str:
+        return self._definition
+
+    @definition.setter
+    def definition(self, value: str) -> None:
+        self._definition = value
+        # Update the instruction for both prompts with the new definition
+        instruction = f"Evaluate the Input based on the criterial defined. Use only 'Yes' (1) and 'No' (0) as verdict.\nCriteria Definition: {self._definition}"
+        self.single_turn_prompt.instruction = instruction
+        self.multi_turn_prompt.instruction = instruction
 
     def _compute_score(
         self, safe_loaded_responses: t.List[AspectCriticOutput]

src/ragas/metrics/_simple_criteria.py

@@ -118,12 +118,12 @@ def __init__(
             _required_columns=required_columns,
         )
 
-        self.definition = definition
+        self._definition = definition
         self.single_turn_prompt = single_turn_prompt or SingleTurnSimpleCriteriaPrompt()
         self.multi_turn_prompt = multi_turn_prompt or MultiTurnSimpleCriteriaPrompt()
 
         # update the instruction for the prompts with the definition
-        instruction = f"Evaluate the Input based on the criterial defined. Give a score between 0 and 5.\nCriteria Definition: {self.definition}"
+        instruction = f"Evaluate the Input based on the criterial defined. Give a score between 0 and 5.\nCriteria Definition: {self._definition}"
         self.single_turn_prompt.instruction = instruction
         self.multi_turn_prompt.instruction = instruction
 
@@ -134,7 +134,19 @@ def __init__(
         )
 
     def __repr__(self) -> str:
-        return f"{self.name}(required_columns={self.required_columns}, llm={self.llm}, definition={self.definition})"
+        return f"{self.name}(required_columns={self.required_columns}, llm={self.llm}, definition={self._definition})"
+
+    @property
+    def definition(self) -> str:
+        return self._definition
+
+    @definition.setter
+    def definition(self, value: str) -> None:
+        self._definition = value
+        # Update the instruction for both prompts with the new definition
+        instruction = f"Evaluate the Input based on the criterial defined. Give a score between 0 and 5.\nCriteria Definition: {self._definition}"
+        self.single_turn_prompt.instruction = instruction
+        self.multi_turn_prompt.instruction = instruction
 
     def _compute_score(
         self, safe_loaded_responses: t.List[SimpleCriteriaOutput]

src/ragas/testset/synthesizers/testset_schema.py

@@ -40,7 +40,7 @@ class TestsetPacket(BaseModel):
     """
 
     samples_original: t.List[TestsetSample]
-    run_id: str = Field(default_factory=lambda: str(uuid4()))
+    run_id: str
     created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
 
 
@@ -56,6 +56,7 @@ class Testset(RagasDataset[TestsetSample]):
     """
 
     samples: t.List[TestsetSample]
+    run_id: str = field(default_factory=lambda: str(uuid4()), repr=False, compare=False)
     cost_cb: t.Optional[CostCallbackHandler] = field(default=None, repr=False)
 
     def to_evaluation_dataset(self) -> EvaluationDataset:
@@ -137,7 +138,7 @@ def total_cost(
     def upload(self, base_url: str = RAGAS_API_URL, verbose: bool = True) -> str:
         import requests
 
-        packet = TestsetPacket(samples_original=self.samples)
+        packet = TestsetPacket(samples_original=self.samples, run_id=self.run_id)
         response = requests.post(
             f"{base_url}/alignment/testset", json=packet.model_dump()
         )

tests/unit/test_metric.py

@@ -1,7 +1,10 @@
 import typing as t
 from dataclasses import dataclass, field
 
+import pytest
+
 from ragas.dataset_schema import SingleTurnSample
+from ragas.metrics import AspectCritic, SimpleCriteriaScore
 from ragas.metrics.base import MetricType
 
 
@@ -89,3 +92,20 @@ async def _single_turn_ascore(self, sample: SingleTurnSample, callbacks):
             user_input="a", response="b", retrieved_contexts=["c"]
         ).to_dict()
     )
+
+
+@pytest.mark.parametrize("metric", [AspectCritic, SimpleCriteriaScore])
+def test_metrics_with_definition(metric):
+    """
+    Test the general metrics like AspectCritic, SimpleCriteriaScore
+    """
+
+    m = metric(name="metric", definition="test")
+
+    # check if the definition is set
+    assert m.definition == "test"
+
+    # check if the definition is updated and the instruction along with it
+    m.definition = "this is a new definition"
+    assert m.definition == "this is a new definition"
+    assert "this is a new definition" in m.single_turn_prompt.instruction