Skip to content

Commit d1dfa38

Browse files
committed
fixes: instruction format (explodinggradients#1672)
New formating example. ``` Given a input and response. Evaluate the submission only using the given criteria. Use only 'Yes' (1) and 'No' (0) as verdict. Please return the output in a JSON format that complies with the following schema as specified in JSON Schema: {'properties': {'reason': {'description': 'Reason for the verdict', 'title': 'Reason', 'type': 'string'}, 'verdict': {'description': 'The verdict (0 or 1) for the submission', 'title': 'Verdict', 'type': 'integer'}}, 'required': ['reason', 'verdict'], 'title': 'AspectCriticOutput', 'type': 'object'} --------EXAMPLES----------- Example 1 Input: { "user_input": "Who was the director of Los Alamos Laboratory?", "response": "Einstein was the director of Los Alamos Laboratory.", "criteria": "Is the output written in perfect grammar" } Output: { "reason": "the criteria for evaluation is whether the output is written in perfect grammar. In this case, the output is grammatically correct.", "verdict": 1 } ----------------------------- Now perform the same with the following input Input: { "user_input": "What is x", "response": "Y", "criteria": "no one" } Output: ```
1 parent c05399d commit d1dfa38

File tree

1 file changed

+12
-17
lines changed

1 file changed

+12
-17
lines changed

src/ragas/prompt/pydantic_prompt.py

+12-17
Original file line numberDiff line numberDiff line change
@@ -42,48 +42,43 @@ def _generate_instruction(self) -> str:
4242
def _generate_output_signature(self, indent: int = 4) -> str:
4343
return (
4444
f"Please return the output in a JSON format that complies with the "
45-
f"following schema as specified in JSON Schema and OpenAPI specification:\n"
45+
f"following schema as specified in JSON Schema:\n"
4646
f"{self.output_model.model_json_schema()}"
4747
)
4848

4949
def _generate_examples(self):
5050
if self.examples:
5151
example_strings = []
52-
for e in self.examples:
52+
for idx, e in enumerate(self.examples):
5353
input_data, output_data = e
5454
example_strings.append(
55-
self.instruction
56-
+ "\n"
57-
+ "input: "
55+
f"Example {idx + 1}\n"
56+
+ "Input: "
5857
+ input_data.model_dump_json(indent=4)
5958
+ "\n"
60-
+ "output: "
59+
+ "Output: "
6160
+ output_data.model_dump_json(indent=4)
6261
)
6362

64-
return (
65-
"These are some examples to show how to perform the above instruction\n"
66-
+ "\n\n".join(example_strings)
67-
)
63+
return "\n--------EXAMPLES-----------\n" + "\n\n".join(example_strings)
6864
# if no examples are provided
6965
else:
7066
return ""
7167

7268
def to_string(self, data: t.Optional[InputModel] = None) -> str:
7369
return (
74-
self._generate_instruction()
75-
+ "\n"
70+
f"{self.instruction}\n"
7671
+ self._generate_output_signature()
7772
+ "\n"
7873
+ self._generate_examples()
79-
+ "\nNow perform the above instruction with the following input\n"
74+
+ "\n-----------------------------\n"
75+
+ "\nNow perform the same with the following input\n"
8076
+ (
81-
"input: " + data.model_dump_json(indent=4) + "\n"
77+
"Input: " + data.model_dump_json(indent=4) + "\n"
8278
if data is not None
83-
else "input: (None)\n"
79+
else "Input: (None)\n"
8480
)
85-
+ "Respond only with a valid JSON object that complies with the specified schema.\n"
86-
+ "output: "
81+
+ "Output: "
8782
)
8883

8984
async def generate(

0 commit comments

Comments
 (0)